diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 0b2f75e352b..c20241136b3 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -113,6 +113,7 @@ from ..common_utils import ( refusal_provider_specific_fields, requires_native_compaction_beta, strip_advisor_blocks_from_messages, + tools_marked_for_a_tool_config_point, ) if TYPE_CHECKING: @@ -2357,6 +2358,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Remove internal LiteLLM parameters that should not be sent to Anthropic API optional_params.pop("is_vertex_request", None) optional_params.pop("client_metadata", None) + request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature + dict[str, object], optional_params + ) + marked_tools: Final = tools_marked_for_a_tool_config_point( + request_params.get("tools"), request_params.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + request_litellm_params: Final = cast( # cast-ok: litellm_params is the untyped dict of the shared signature + Mapping[str, object], litellm_params + ) + AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1) # ``top_k`` is a provider-specific kwarg that bypasses # ``map_openai_params``; gate it here, the single boundary shared by @@ -2376,6 +2390,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "model": model, "messages": anthropic_messages, **optional_params, + **({} if marked_tools is None else {"tools": list(marked_tools)}), } self._apply_output_config(data=data, model=model, optional_params=optional_params) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index dd9b22f3d5d..578329c4c11 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -50,6 +50,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_OAUTH_TOKEN_PREFIX, ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, @@ -111,7 +112,7 @@ _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$") _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) -_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _DECODED_JSON: Final = TypeAdapter(object) @@ -160,6 +161,40 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None ) +def _tool_config_point_control(points: object) -> Mapping[object, object] | None: + point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ()) + tool_config_point: Final = next( + (fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None + ) + if tool_config_point is None: + return None + control: Final = tool_config_point.get("control") + return {"type": "ephemeral"} if control is None else validated_object_mapping(control) + + +def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None: + """Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches. + + Anthropic caches tools as one prefix ending at the marked tool, so the last one carries + the mark, skipping tool-search tools whose transform drops cache markers. A client that + already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked. + """ + control: Final = _tool_config_point_control(points) + tool_list: Final = validated_object_list(tools) or () + tool_fields: Final = tuple(validated_object_mapping(tool) for tool in tool_list) + if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields): + return None + cacheable: Final = tuple( + (position, fields) + for position, fields in enumerate(tool_fields) + if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + ) + if not cacheable: + return None + position, fields = cacheable[-1] + return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :]) + + def is_claude_code_user_agent(user_agent: str) -> bool: """Claude Code sends its API calls through the Anthropic SDK as `claude-cli/` and its own fetches, such as gateway model discovery, as `claude-code/`""" @@ -173,9 +208,9 @@ def validated_object_mapping(value: object) -> dict[object, object] | None: return None -def _validated_claude_code_list(value: object) -> list[object] | None: +def validated_object_list(value: object) -> Sequence[object] | None: try: - return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value) + return OBJECT_LIST_ADAPTER.validate_python(value) except ValidationError: return None @@ -202,7 +237,7 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None: if isinstance(system, str): return (system,) - blocks: Final = _validated_claude_code_list(system) + blocks: Final = validated_object_list(system) if blocks is None: return None block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks) @@ -1963,8 +1998,8 @@ def tool_call_is_rebuilt_as_server_tool_use( return ( find_anthropic_server_tool_result( tool_call_id, - _validated_claude_code_list(fields.get("web_search_results")), - _validated_claude_code_list(fields.get("tool_results")), + validated_object_list(fields.get("web_search_results")), + validated_object_list(fields.get("tool_results")), ) is not None ) diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 00bf1774dd0..eeff6302ee8 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -14,10 +14,12 @@ from typing import Final, cast from pydantic import TypeAdapter import litellm +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( strip_provider_specific_fields_from_anthropic_messages, + tools_marked_for_a_tool_config_point, ) from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -667,7 +669,13 @@ def anthropic_messages_handler( f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}" ) - local_vars.update(kwargs) + request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped + marked_tools: Final = tools_marked_for_a_tool_config_point( + local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) + local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})}) anthropic_messages_optional_request_params: Final = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( params=local_vars, diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 564dc1db527..636bb0b84ee 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -23,7 +23,7 @@ class CacheControlMessageInjectionPoint(TypedDict): class CacheControlToolConfigInjectionPoint(TypedDict): - """Type for tool_config-level injection points (Bedrock).""" + """Type for tool_config-level injection points (Anthropic and Bedrock).""" location: Literal["tool_config"] control: ChatCompletionCachedContent | None diff --git a/litellm/utils.py b/litellm/utils.py index 5e4ed4ffadc..3d6c602f4b7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5174,6 +5174,9 @@ def get_optional_params( EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"}) +LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",)) + + def add_provider_specific_params_to_optional_params( optional_params: dict, passed_params: dict, @@ -5197,7 +5200,7 @@ def add_provider_specific_params_to_optional_params( if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) for k in passed_params: - if k not in openai_params and passed_params[k] is not None: + if k not in openai_params and k not in LITELLM_ONLY_PARAMS and passed_params[k] is not None: extra_body[k] = passed_params[k] if not isinstance(optional_params.get("extra_body"), dict): optional_params["extra_body"] = {} diff --git a/tests/integration/providers/test_tool_config_cache_point_wire.py b/tests/integration/providers/test_tool_config_cache_point_wire.py new file mode 100644 index 00000000000..6373e122975 --- /dev/null +++ b/tests/integration/providers/test_tool_config_cache_point_wire.py @@ -0,0 +1,312 @@ +import json +import uuid +from typing import Final +from urllib.parse import unquote, urlsplit + +from integration._support.client import Gateway +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_ANTHROPIC_MODEL: Final = "claude-sonnet-4-6" +_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-6" +_OPENAI_MODEL: Final = "gpt-4.1" +_GEMINI_MODEL: Final = "gemini-3.8-flash" +_NOVA_MODEL: Final = "us.amazon.nova-pro-v1:0" +_API_KEY: Final = "synthetic-provider-key" +_CONTROL: Final = {"type": "ephemeral", "ttl": "1h"} +_TOOL_CONFIG_POINT: Final = [{"location": "tool_config", "control": _CONTROL}] +_AWS: Final = { + "aws_access_key_id": "AKIAINTEGRATIONTEST", + "aws_secret_access_key": "integration-secret", + "aws_region_name": "us-east-1", +} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_TOOL_LIST: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _openai_tools() -> list[JsonValue]: + return [ + { + "type": "function", + "function": { + "name": name, + "description": f"{name} tool", + "parameters": {"type": "object", "properties": {}}, + }, + } + for name in ("get_weather", "get_forecast") + ] + + +def _anthropic_tools() -> list[JsonValue]: + return [ + {"name": name, "description": f"{name} tool", "input_schema": {"type": "object", "properties": {}}} + for name in ("get_weather", "get_forecast") + ] + + +def _anthropic_reply(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "type": "message", + "role": "assistant", + "model": _ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3, "cache_creation_input_tokens": 10}, + } + ).encode() + + +def _converse_reply() -> bytes: + return json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": "done"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 12, "outputTokens": 3, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + } + ).encode() + + +def _assert_last_anthropic_tool_marked(body: dict[str, JsonValue]) -> None: + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(body["tools"]) + assert [tool["name"] for tool in tools] == ["get_weather", "get_forecast"], tools + assert "cache_control" not in tools[0], tools + assert tools[1]["cache_control"] == _CONTROL, tools + + +def test_chat_completions_tool_config_point_marks_the_last_anthropic_tool(gateway: Gateway) -> None: + identity: Final = f"anthropic-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + _assert_last_anthropic_tool_marked(_JSON_OBJECT.validate_json(request.body)) + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_ANTHROPIC_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _openai_tools(), + }, + ) + assert response.status_code == 200, response.text + assert response.json()["usage"]["cache_creation_input_tokens"] == 10, response.text + assert len(wire.drain()) == 1 + + +def test_messages_tool_config_point_marks_the_last_anthropic_tool(gateway: Gateway) -> None: + identity: Final = f"anthropic-native-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + _assert_last_anthropic_tool_marked(_JSON_OBJECT.validate_json(request.body)) + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_ANTHROPIC_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert response.json()["usage"]["cache_creation_input_tokens"] == 10, response.text + assert len(wire.drain()) == 1 + + +def test_chat_completions_tool_config_point_appends_a_bedrock_converse_cache_point(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + path: Final = unquote(urlsplit(request.target).path) + assert request.method == "POST" and path == f"/model/{_BEDROCK_MODEL}/converse", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"]) + assert [tuple(tool) for tool in tools] == [("toolSpec",), ("toolSpec",), ("cachePoint",)], tools + assert tools[2] == {"cachePoint": {"type": "default", "ttl": "1h"}}, tools + return Reply(body=_converse_reply()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{_BEDROCK_MODEL}", + api_base=wire.url, + api_key=None, + cache_control_injection_points=_TOOL_CONFIG_POINT, + **_AWS, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _openai_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_chat_completions_tool_config_point_never_reaches_openai(gateway: Gateway) -> None: + identity: Final = f"openai-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/chat/completions", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(body["tools"]) + assert [tuple(sorted(tool)) for tool in tools] == [("function", "type"), ("function", "type")], tools + return chat_reply(identity, _OPENAI_MODEL, "done", stream=False) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{_OPENAI_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _openai_tools(), + }, + ) + assert response.status_code == 200, response.text + assert response.json()["id"] == identity, response.text + assert len(wire.drain()) == 1 + + +def _gemini_reply() -> bytes: + return json.dumps( + { + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "done"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 12, "candidatesTokenCount": 3, "totalTokenCount": 15}, + "modelVersion": _GEMINI_MODEL, + } + ).encode() + + +def test_messages_tool_config_point_never_reaches_openai_through_the_responses_bridge(gateway: Gateway) -> None: + identity: Final = f"openai-bridge-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/responses", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + return responses_reply(identity, _OPENAI_MODEL, "done", stream=False) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{_OPENAI_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_messages_tool_config_point_never_reaches_gemini_through_the_chat_bridge(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + assert request.method == "POST" and path == f"/models/{_GEMINI_MODEL}:generateContent", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + return Reply(body=_gemini_reply()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"gemini/{_GEMINI_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_messages_tool_config_point_appends_a_bedrock_converse_cache_point_through_the_chat_bridge( + gateway: Gateway, +) -> None: + def respond(request: Request) -> Reply: + path: Final = unquote(urlsplit(request.target).path) + assert request.method == "POST" and path == f"/model/{_NOVA_MODEL}/converse", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"]) + assert [tuple(tool) for tool in tools] == [("toolSpec",), ("toolSpec",), ("cachePoint",)], tools + assert tools[2] == {"cachePoint": {"type": "default"}}, tools + return Reply(body=_converse_reply()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{_NOVA_MODEL}", + api_base=wire.url, + api_key=None, + cache_control_injection_points=_TOOL_CONFIG_POINT, + **_AWS, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 724e137c835..9e70d213f47 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1294,7 +1294,6 @@ def test_anthropic_structured_output_beta_header(): print(f"raw_request_headers: {response['raw_request_headers']}") assert "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"] - @pytest.mark.parametrize( "model_name", [ @@ -1346,6 +1345,7 @@ def test_opus_uses_native_structured_output(model_name): assert optional_params.get("json_mode") is True + def test_native_structured_output_uses_bundled_capability_when_remote_map_lags( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -3636,7 +3636,9 @@ def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list return sorted(block[id_key] for block in message["content"] if block["type"] == block_type) -_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn([], {"type": "text", "text": "I have no tool that books flights."}) +_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn( + [], {"type": "text", "text": "I have no tool that books flights."} +) @pytest.mark.parametrize( @@ -9445,7 +9447,9 @@ def test_map_openai_params_runs_tool_search_on_a_custom_named_deployment_whose_m AzureAnthropicConfig().map_openai_params( non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False ) - litellm.register_model({"azure_ai/my-claude-deployment": {"supports_tool_search": True, "litellm_provider": "azure_ai"}}) + litellm.register_model( + {"azure_ai/my-claude-deployment": {"supports_tool_search": True, "litellm_provider": "azure_ai"}} + ) mapped: Final = AzureAnthropicConfig().map_openai_params( non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False ) @@ -9559,3 +9563,81 @@ def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> Non ) provider_specific_fields = result.choices[0].message.provider_specific_fields or {} assert "stop_details" not in provider_specific_fields + + +def _two_function_tools() -> list[dict]: + return [ + { + "type": "function", + "function": {"name": f"tool_{i}", "description": f"Tool {i}", "parameters": {"type": "object"}}, + } + for i in range(2) + ] + + +@pytest.mark.parametrize( + "control, expected_control", + [ + (None, {"type": "ephemeral"}), + ({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}), + ], +) +def test_transform_request_tool_config_point_marks_last_tool_and_drops_the_key(control, expected_control): + """LIT-9880: a tool_config injection point used to be spread into the Anthropic body, which + Anthropic rejected with 400 Extra inputs are not permitted.""" + tools = _two_function_tools() + tools_before = copy.deepcopy(tools) + point = {"location": "tool_config", **({"control": control} if control is not None else {})} + litellm_params = {"metadata": {}, "model_info": {"id": "deployment-1"}} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [point]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][-1]["cache_control"] == expected_control + assert "cache_control" not in data["tools"][0] + assert tools == tools_before + assert litellm_params["metadata"]["litellm_gateway_injected_cache"] == "deployment-1" + + +def test_transform_request_tool_config_point_keeps_the_clients_tool_marker(): + tools = _two_function_tools() + tools[0]["cache_control"] = {"type": "ephemeral"} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][1] + + +def test_transform_request_tool_config_point_skips_tool_search_tools(): + tool_search_type = next(iter(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)) + tools = [*_two_function_tools(), {"type": tool_search_type, "name": "tool_search"}] + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert data["tools"][1]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][2] + assert "cache_control" not in data["tools"][0] + + +def test_transform_request_tool_config_point_without_tools_drops_the_key(): + litellm_params = {"metadata": {}} + + data = _make_transform_request( + optional_params={"cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert "tools" not in data + assert litellm_params["metadata"] == {} diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 0b68f54c504..bdc144b91cc 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1,4 +1,5 @@ import asyncio +import copy import json import os import uuid @@ -1483,6 +1484,7 @@ async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_defaul assert seen_urls == ["https://deepseek.internal.example/anthropic/v1/messages"] + @pytest.mark.asyncio async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" @@ -1597,7 +1599,9 @@ def _claude_code_auto_mode_request() -> tuple[list[dict[str, object]], list[dict return safeguards, safeguard_results -def _upstream_answering_with(safeguard_results: list[dict[str, object]], captured: dict[str, object]) -> AsyncHTTPHandler: +def _upstream_answering_with( + safeguard_results: list[dict[str, object]], captured: dict[str, object] +) -> AsyncHTTPHandler: def upstream_records_the_request(request: httpx.Request) -> httpx.Response: captured["body"] = json.loads(request.content) captured["anthropic-beta"] = request.headers.get("anthropic-beta") @@ -1623,7 +1627,9 @@ def _upstream_answering_with(safeguard_results: list[dict[str, object]], capture _CLIENT_BETA_HEADERS: Final = ( - pytest.param({"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"), + pytest.param( + {"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta" + ), pytest.param({"anthropic-beta": "interleaved-thinking-2025-05-14"}, id="client_omits_beta"), pytest.param({}, id="client_sends_no_beta_header"), ) @@ -1748,7 +1754,11 @@ def _tool_use_sse(model: str) -> bytes: "delta": {"type": "input_json_delta", "partial_json": json.dumps(_CURL_TOOL_USE["input"])}, }, {"type": "content_block_stop", "index": 0}, - {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 4}}, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 4}, + }, {"type": "message_stop"}, ) return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() @@ -1902,26 +1912,20 @@ async def test_anthropic_messages_litellm_router_latency_metadata_tracking(): print("Call kwargs:", json.dumps(call_kwargs, indent=2, default=str)) # Verify that litellm_metadata was passed and contains _latency_per_deployment - assert ( - "litellm_metadata" in call_kwargs - ), "litellm_metadata should be passed to anthropic_messages" + assert "litellm_metadata" in call_kwargs, "litellm_metadata should be passed to anthropic_messages" litellm_metadata = call_kwargs["litellm_metadata"] assert litellm_metadata is not None, "litellm_metadata should not be None" - assert isinstance( - litellm_metadata, dict - ), "litellm_metadata should be a dictionary" + assert isinstance(litellm_metadata, dict), "litellm_metadata should be a dictionary" # Verify _latency_per_deployment is present - assert ( - "_latency_per_deployment" in litellm_metadata - ), "litellm_metadata should contain _latency_per_deployment field" + assert "_latency_per_deployment" in litellm_metadata, ( + "litellm_metadata should contain _latency_per_deployment field" + ) # Verify the structure of _latency_per_deployment latency_per_deployment = litellm_metadata["_latency_per_deployment"] - assert isinstance( - latency_per_deployment, dict - ), "_latency_per_deployment should be a dictionary" + assert isinstance(latency_per_deployment, dict), "_latency_per_deployment should be a dictionary" print(f"✅ Latency per deployment data: {latency_per_deployment}") @@ -2079,9 +2083,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): when using anthropic.messages.acreate with a bedrock model """ # Mock the get_credentials method - with unittest.mock.patch.object( - BaseAWSLLM, "get_credentials" - ) as mock_get_credentials: + with unittest.mock.patch.object(BaseAWSLLM, "get_credentials") as mock_get_credentials: # Create a proper mock for credentials with the necessary attributes mock_credentials = unittest.mock.MagicMock() mock_credentials.access_key = "mock_access_key" @@ -2092,9 +2094,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): # We also need to mock the actual AWS request signing to avoid real API calls with unittest.mock.patch("botocore.auth.SigV4Auth.add_auth"): # Set up mock for AsyncHTTPHandler.post to avoid actual API calls - with unittest.mock.patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post" - ) as mock_post: + with unittest.mock.patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") as mock_post: # Configure mock response mock_response = unittest.mock.MagicMock() mock_response.raise_for_status = unittest.mock.MagicMock() @@ -2136,9 +2136,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): # Assert that our test credentials were passed correctly for param_name, param_value in aws_params.items(): - assert ( - call_args[param_name] == param_value - ), f"Parameter {param_name} was not passed correctly" + assert call_args[param_name] == param_value, f"Parameter {param_name} was not passed correctly" @pytest.mark.asyncio @@ -2166,11 +2164,8 @@ async def test_anthropic_messages_bedrock_dynamic_region(): # Patch necessary AWS components with ( unittest.mock.patch("botocore.auth.SigV4Auth.add_auth"), - unittest.mock.patch.object( - BaseAWSLLM, "get_credentials" - ) as mock_get_credentials, + unittest.mock.patch.object(BaseAWSLLM, "get_credentials") as mock_get_credentials, ): - # Setup mock credentials mock_credentials = unittest.mock.MagicMock() mock_credentials.access_key = "test_access_key" @@ -2199,11 +2194,52 @@ async def test_anthropic_messages_bedrock_dynamic_region(): # Check that the URL contains the correct region url = call_args.kwargs.get("url", "") - assert ( - f"bedrock-runtime.{test_region}.amazonaws.com" in url - ), f"URL does not contain the correct region. URL: {url}" + assert f"bedrock-runtime.{test_region}.amazonaws.com" in url, ( + f"URL does not contain the correct region. URL: {url}" + ) # Verify get_credentials was called with the correct region mock_get_credentials.assert_called_once() credentials_args = mock_get_credentials.call_args.kwargs assert credentials_args.get("aws_region_name") == test_region + + + + +def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch): + """LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the + native path dropped it silently, so the tools went out without any cache breakpoint.""" + from litellm.llms.anthropic.pass_through.messages import handler + + captured = {} + + def fake_native(**kwargs): + captured["params"] = kwargs["anthropic_messages_optional_request_params"] + captured["kwargs"] = kwargs["kwargs"] + captured["messages"] = kwargs["messages"] + return "native-passthrough" + + monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native) + tools = [{"name": f"tool_{i}", "description": f"Tool {i}", "input_schema": {"type": "object"}} for i in range(2)] + tools_before = copy.deepcopy(tools) + + result = handler.anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="anthropic/claude-opus-4-6", + api_key="sk-test", + tools=tools, + litellm_metadata={}, + cache_control_injection_points=[ + {"location": "message", "role": "user"}, + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}, + ], + ) + + assert result == "native-passthrough" + assert "cache_control_injection_points" not in captured["params"] + assert "cache_control_injection_points" not in captured["kwargs"] + assert captured["params"]["tools"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert "cache_control" not in captured["params"]["tools"][0] + assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert tools == tools_before diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 61b45d7d388..83c6cff1e75 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1452,8 +1452,6 @@ def test_get_model_info_bedrock_inference_profile_arn_keeps_regional_row(local_m assert bare["input_cost_per_token"] == base["input_cost_per_token"] - - @pytest.mark.parametrize( "bare_key", [ @@ -2289,9 +2287,7 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch: def test_utils_defines_tests_without_invoking_them_at_module_scope() -> None: tree: Final = ast.parse(Path(__file__).read_text()) defined: Final = frozenset( - node.name - for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) ) invoked: Final = tuple( node.value.func.id @@ -3525,9 +3521,7 @@ class TestExtraBodyCannotOverrideModel: } ] - untouched = litellm.get_optional_params( - model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools() - ) + untouched = litellm.get_optional_params(model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools()) assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched result = litellm.get_optional_params( @@ -4567,7 +4561,9 @@ def test_is_prompt_caching_valid_prompt_stops_counting_once_the_minimum_is_reach assert is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=1024) is True assert sum(counted_messages) < len(long_prompt), sum(counted_messages) - full_count = litellm.token_counter(model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True) + full_count = litellm.token_counter( + model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True + ) assert ( is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=full_count) is True @@ -5034,7 +5030,13 @@ _SUCCESS_RESPONSES_BY_CALL_TYPE: Final = ( pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"), pytest.param( ResponsesAPIResponse( - id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6" + id="resp_abc", + created_at=1, + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + model="gpt-5.6", ), CallTypes.aresponses, id="responses", @@ -5062,7 +5064,9 @@ async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hoo assert result is response assert second_hook.seen_responses == (response,) - failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message) + failure_logs: Final = tuple( + r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message + ) assert len(failure_logs) == 1 assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message assert str(call_type) in failure_logs[0].message @@ -5199,17 +5203,13 @@ async def test_wrapper_async_logs_converted_responses_stream_with_standard_loggi ({"metadata": None}, True, False), ], ) -def test_is_litellm_router_call_is_async_aware( - kwargs: Mapping[str, object], is_async: bool, expected: bool -) -> None: +def test_is_litellm_router_call_is_async_aware(kwargs: Mapping[str, object], is_async: bool, expected: bool) -> None: assert _is_litellm_router_call(kwargs, is_async=is_async) is expected @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) -async def test_router_aresponses_does_not_run_sdk_retries( - monkeypatch: pytest.MonkeyPatch, stream: bool -) -> None: +async def test_router_aresponses_does_not_run_sdk_retries(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() model_list: Final = [ @@ -5223,9 +5223,7 @@ async def test_router_aresponses_does_not_run_sdk_retries( }, } ] - router: Final = litellm.Router( - model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True - ) + router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True) try: with respx.mock(assert_all_called=True) as respx_mock: @@ -5261,9 +5259,7 @@ def test_router_responses_keeps_sdk_retries_for_sync_router_call( }, } ] - router: Final = litellm.Router( - model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True - ) + router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True) try: with respx.mock(assert_all_called=True) as respx_mock: @@ -7148,7 +7144,9 @@ def test_function_setup_logs_the_search_query_edit_prompt_and_ocr_document_summa @pytest.mark.parametrize("original_function", ("atext_completion", "text_completion")) -def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation(original_function: str) -> None: +def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation( + original_function: str, +) -> None: assert _logged_request_messages(original_function, model="gpt-4o") is None @@ -7190,6 +7188,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def setup_and_teardown(): """ @@ -7203,6 +7202,7 @@ def setup_and_teardown(): loop.close() asyncio.set_event_loop(None) + MODEL: Final = "anthropic/claude-haiku-4-5" @@ -7212,6 +7212,7 @@ def test_validate_tool_choice_none(): result = validate_chat_completion_tool_choice(None, model=MODEL) assert result is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_string(): """Test that string values are returned as-is.""" @@ -7219,6 +7220,7 @@ def test_validate_tool_choice_string(): assert validate_chat_completion_tool_choice("none", model=MODEL) == "none" assert validate_chat_completion_tool_choice("required", model=MODEL) == "required" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_standard_dict(): """Test standard OpenAI format with function.""" @@ -7226,6 +7228,7 @@ def test_validate_tool_choice_standard_dict(): result = validate_chat_completion_tool_choice(tool_choice, model=MODEL) assert result == tool_choice + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_cursor_format(): """Cursor IDE format {"type": "auto"} is unwrapped to the bare string.""" @@ -7233,6 +7236,7 @@ def test_validate_tool_choice_cursor_format(): assert validate_chat_completion_tool_choice({"type": "none"}, model=MODEL) == "none" assert validate_chat_completion_tool_choice({"type": "required"}, model=MODEL) == "required" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize( "tool_choice", @@ -7254,6 +7258,7 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): assert exc_info.value.status_code == 400 assert exc_info.value.model == MODEL + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("tool_choice", [123, []]) def test_validate_tool_choice_invalid_type_is_a_400(tool_choice): @@ -7264,6 +7269,7 @@ def test_validate_tool_choice_invalid_type_is_a_400(tool_choice): validate_chat_completion_tool_choice(tool_choice, model=MODEL) assert exc_info.value.status_code == 400 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_without_model_is_still_a_400(): """Callers that predate the model argument keep getting a 400, with an empty model on the error.""" @@ -7272,12 +7278,14 @@ def test_validate_tool_choice_without_model_is_still_a_400(): assert exc_info.value.status_code == 400 assert exc_info.value.model == "" + @pytest.fixture() def _vcr_outcome_gate_local_testing(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -7330,6 +7338,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -7373,6 +7382,7 @@ def setup_and_teardown_local_testing(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7398,6 +7408,7 @@ def test_vertex_projects(): assert "vertex_ai_project" in optional_params assert "vertex_ai_location" in optional_params + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7413,6 +7424,7 @@ def test_bedrock_embed_v2_regular(): print(f"received optional_params: {optional_params}") assert optional_params == {"dimensions": 512} + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7431,6 +7443,7 @@ def test_bedrock_embed_v2_with_drop_params(): print(f"received optional_params: {optional_params}") assert optional_params == {"dimensions": 512, "embeddingTypes": ["binary"]} + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7452,6 +7465,7 @@ def test_openai_non_text_embedding_3_with_allowed_openai_params(): print(f"received optional_params: {optional_params}") assert optional_params.get("dimensions") == 1024 + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7506,6 +7520,8 @@ def test_openai_non_text_embedding_3_drop_params_per_call(): finally: litellm.drop_params = prev_drop_params + + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7533,12 +7549,14 @@ def test_openai_non_text_embedding_3_drop_params_global(): finally: litellm.drop_params = prev_drop_params + @pytest.fixture() def _vcr_outcome_gate_search_tests(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.mark.usefixtures("_vcr_outcome_gate_search_tests") def test_search_tool_name_in_all_litellm_params(): """ @@ -7548,6 +7566,7 @@ def test_search_tool_name_in_all_litellm_params(): """ assert "search_tool_name" in all_litellm_params + @pytest.mark.usefixtures("_vcr_outcome_gate_search_tests") def test_filter_out_search_tool_name(): """ @@ -7574,6 +7593,7 @@ def test_filter_out_search_tool_name(): assert filtered["query"] == "latest ai developments" assert filtered["max_results"] == 5 + @pytest.mark.asyncio async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: pytest.MonkeyPatch) -> None: """Chat over the Responses bridge exits two @client wrappers with one logging object. Issue @@ -7726,9 +7746,7 @@ def test_trimming_with_system_message_within_max_tokens(): "content": "This is a medium normal message, let's say litellm is awesome.", }, ] - trimmed_messages = trim_messages( - messages, max_tokens=30, model="gpt-4-0613" - ) + trimmed_messages = trim_messages(messages, max_tokens=30, model="gpt-4-0613") assert len(trimmed_messages) == 2 assert trimmed_messages[0]["content"] == "This is a short system message" @@ -7911,9 +7929,7 @@ def test_aget_valid_models(): print(valid_models) # list of openai supported llms on litellm - expected_models = ( - litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models - ) + expected_models = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models assert set(valid_models) == set(expected_models) @@ -8002,9 +8018,7 @@ def test_get_chat_completion_prompt(): prompt_variables=None, ) - assert litellm_logging_obj.messages == [ - {"role": "user", "content": updated_message} - ] + assert litellm_logging_obj.messages == [{"role": "user", "content": updated_message}] def test_redact_msgs_from_logs(): @@ -8076,9 +8090,7 @@ def test_redact_embedding_response(): litellm.turn_off_message_logging = True # Create a test EmbeddingResponse with usage data - original_usage = litellm.Usage( - prompt_tokens=10, completion_tokens=0, total_tokens=10 - ) + original_usage = litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10) original_data = [ {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3, 0.4, 0.5]}, {"object": "embedding", "index": 1, "embedding": [0.6, 0.7, 0.8, 0.9, 1.0]}, @@ -8114,9 +8126,7 @@ def test_redact_embedding_response(): # Assert the redacted response preserves critical metadata assert _redacted_response_obj.usage == original_usage # usage should be preserved - assert ( - _redacted_response_obj.model == "text-embedding-3-small" - ) # model should be preserved + assert _redacted_response_obj.model == "text-embedding-3-small" # model should be preserved assert _redacted_response_obj.object == "list" # object should be preserved # Assert sensitive data is cleared @@ -8170,12 +8180,8 @@ def test_redact_msgs_from_logs_with_dynamic_params(): ) # Test Case 1: standard_callback_dynamic_params = False (or not set) - standard_callback_dynamic_params = StandardCallbackDynamicParams( - turn_off_message_logging=False - ) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=False) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -8184,12 +8190,8 @@ def test_redact_msgs_from_logs_with_dynamic_params(): assert _redacted_response_obj.choices[0].message.content == test_content # Test Case 2: standard_callback_dynamic_params = True - standard_callback_dynamic_params = StandardCallbackDynamicParams( - turn_off_message_logging=True - ) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=True) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -8200,9 +8202,7 @@ def test_redact_msgs_from_logs_with_dynamic_params(): # Test Case 3: standard_callback_dynamic_params does not set turn_off_message_logging # since litellm.turn_off_message_logging is True redaction should occur standard_callback_dynamic_params = StandardCallbackDynamicParams() - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -8544,10 +8544,7 @@ def test_models_by_provider(): continue elif k == "sample_spec": continue - elif ( - v["litellm_provider"] == "sagemaker" - or v["litellm_provider"] == "bedrock_converse" - ): + elif v["litellm_provider"] == "sagemaker" or v["litellm_provider"] == "bedrock_converse": continue elif v.get("mode") in ("search", "decisions", "evaluation"): continue @@ -8555,9 +8552,7 @@ def test_models_by_provider(): providers.add(v["litellm_provider"]) for provider in providers: - assert provider in models_by_provider.keys() or JSONProviderRegistry.exists( - provider - ) + assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(provider) @pytest.fixture @@ -8579,16 +8574,11 @@ def restore_end_user_cost_tracking_flags(monkeypatch: pytest.MonkeyPatch) -> Non ({"user_api_key_end_user_id": "123"}, True, None), ], ) -def test_get_end_user_id_for_cost_tracking( - litellm_params, disable_end_user_cost_tracking, expected_end_user_id -): +def test_get_end_user_id_for_cost_tracking(litellm_params, disable_end_user_cost_tracking, expected_end_user_id): from litellm.utils import get_end_user_id_for_cost_tracking litellm.disable_end_user_cost_tracking = disable_end_user_cost_tracking - assert ( - get_end_user_id_for_cost_tracking(litellm_params=litellm_params) - == expected_end_user_id - ) + assert get_end_user_id_for_cost_tracking(litellm_params=litellm_params) == expected_end_user_id @pytest.mark.usefixtures("restore_end_user_cost_tracking_flags") @@ -8605,13 +8595,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ): from litellm.utils import get_end_user_id_for_cost_tracking - litellm.enable_end_user_cost_tracking_prometheus_only = ( - enable_end_user_cost_tracking_prometheus_only - ) + litellm.enable_end_user_cost_tracking_prometheus_only = enable_end_user_cost_tracking_prometheus_only assert ( - get_end_user_id_for_cost_tracking( - litellm_params=litellm_params, service_type="prometheus" - ) + get_end_user_id_for_cost_tracking(litellm_params=litellm_params, service_type="prometheus") == expected_end_user_id ) @@ -8626,20 +8612,14 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ), # Test with only litellm_metadata field (new behavior) ( - { - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - } - }, + {"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}}, "user_from_litellm_metadata", ), # Test with both fields - metadata should take precedence for user_api_key fields ( { "metadata": {"user_api_key_end_user_id": "user_from_metadata"}, - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - }, + "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, }, "user_from_metadata", ), @@ -8655,9 +8635,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ( { "metadata": {}, - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - }, + "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, }, "user_from_litellm_metadata", ), @@ -8665,9 +8643,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ({}, None), ], ) -def test_get_end_user_id_for_cost_tracking_metadata_handling( - litellm_params, expected_end_user_id -): +def test_get_end_user_id_for_cost_tracking_metadata_handling(litellm_params, expected_end_user_id): """ Test that get_end_user_id_for_cost_tracking correctly handles both metadata and litellm_metadata fields using the get_litellm_metadata_from_kwargs helper function. @@ -8925,9 +8901,7 @@ def test_add_custom_logger_callback_to_specific_event_e2e_failure(monkeypatch): assert len(litellm.success_callback) == curr_len_success_callback assert len(litellm.failure_callback) == curr_len_failure_callback - assert any( - isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback - ) + assert any(isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback) @pytest.mark.asyncio @@ -9024,20 +8998,14 @@ def test_validate_user_messages_invalid_content_type(): [ { "name": "default_on_guardrail", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=True) - ], + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=True)], "kwargs": {"metadata": {"requester_metadata": {"guardrails": []}}}, "expected": ["test_guardrail"], }, { "name": "request_specific_guardrail", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=False) - ], - "kwargs": { - "metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}} - }, + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], + "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}}, "expected": ["test_guardrail"], }, { @@ -9046,18 +9014,12 @@ def test_validate_user_messages_invalid_content_type(): CustomGuardrail(guardrail_name="default_guardrail", default_on=True), CustomGuardrail(guardrail_name="request_guardrail", default_on=False), ], - "kwargs": { - "metadata": { - "requester_metadata": {"guardrails": ["request_guardrail"]} - } - }, + "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["request_guardrail"]}}}, "expected": ["default_guardrail", "request_guardrail"], }, { "name": "empty_metadata", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=False) - ], + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], "kwargs": {}, "expected": [], }, @@ -9836,9 +9798,7 @@ def test_pre_call_rules_see_messages_passed_positionally(monkeypatch: pytest.Mon ) with pytest.raises(litellm.APIResponseValidationError, match="failed post-call-rule check"): - litellm.completion( - "gpt-4o-mini", [{"role": "user", "content": "say something forbidden"}], api_key="sk-test" - ) + litellm.completion("gpt-4o-mini", [{"role": "user", "content": "say something forbidden"}], api_key="sk-test") allowed: Final = litellm.completion("gpt-4o-mini", [{"role": "user", "content": "say hi"}], api_key="sk-test") assert route.call_count == 1 @@ -9856,3 +9816,18 @@ def test_vertex_llama3_model_info_resolves_without_the_meta_prefix(): prefixed_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") assert litellm.get_model_info(model=short_name, custom_llm_provider="vertex_ai") == prefixed_info, model assert prefixed_info["key"] == f"vertex_ai/{model}", model + + +def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body(): + """LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no + step that consumes it, so it must not ride along in extra_body and 400 at OpenAI.""" + optional_params = litellm.utils.get_optional_params( + model="gpt-4.1", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral"}}], + extra_body={"kept": True}, + ) + + assert optional_params["extra_body"] == {"kept": True} + assert "cache_control_injection_points" not in optional_params