From 030dd40a93c32960c90c2213d706a7b8a8614fb8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:03:23 -0700 Subject: [PATCH 01/12] fix(vertex_ai): forward and auto-add the compact-2026-09-04 beta on Claude routes --- litellm/anthropic_beta_headers_config.json | 1 + litellm/llms/anthropic/common_utils.py | 4 +- .../transformation.py | 5 +- .../anthropic/transformation.py | 17 ++- .../anthropic/test_anthropic_common_utils.py | 41 +++++++ ...artner_models_anthropic_messages_config.py | 45 +++++++- ...partner_models_anthropic_transformation.py | 102 ++++++++++++++++++ .../test_anthropic_beta_headers_filtering.py | 14 +-- 8 files changed, 216 insertions(+), 13 deletions(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 332d9b3ad9d..f06976cbf81 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -188,6 +188,7 @@ "bash_20250124": null, "code-execution-2025-08-25": null, "compact-2026-01-12": "compact-2026-01-12", + "compact-2026-09-04": "compact-2026-09-04", "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 6da3807ab15..ba892816466 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -111,13 +111,15 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "vertex_ai"}) + def requires_native_compaction_beta( custom_llm_provider: str, optional_params: Mapping[str, object], messages: Sequence[object], ) -> bool: - return custom_llm_provider == "anthropic" and ( + return custom_llm_provider in NATIVE_COMPACTION_PROVIDERS and ( optional_params.get("compaction") is not None or any( isinstance(block, Mapping) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index be2dacd23c2..125eec8976a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -1,6 +1,6 @@ from typing import Any, Final -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, requires_native_compaction_beta from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, _messages_carry_output_config, @@ -115,6 +115,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if _messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + if requires_native_compaction_beta("vertex_ai", optional_params, messages): + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + if beta_values: headers["anthropic-beta"] = ",".join(beta_values) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 2fab6f438f6..7f9e0a0ea3e 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -7,15 +7,19 @@ import httpx import litellm from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from ....anthropic.chat.transformation import AnthropicConfig +from ....anthropic.common_utils import requires_native_compaction_beta from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +_VERTEX_HEADER_ONLY_BETAS: Final = frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) + class VertexAIError(Exception): def __init__(self, status_code, message): @@ -131,6 +135,9 @@ class VertexAIAnthropicConfig(AnthropicConfig): if context_management: self._add_context_management_beta_headers(beta_set, context_management) + if requires_native_compaction_beta("vertex_ai", optional_params, data["messages"]): + beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + extra_headers: Final = optional_params.get("extra_headers") or {} anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "") if isinstance(anthropic_beta_value, str) and anthropic_beta_value: @@ -143,9 +150,13 @@ class VertexAIAnthropicConfig(AnthropicConfig): data.pop("extra_headers", None) - if beta_set: - data["anthropic_beta"] = list(beta_set) - headers["anthropic-beta"] = ",".join(beta_set) + body_betas: Final = beta_set - _VERTEX_HEADER_ONLY_BETAS + if body_betas: + data["anthropic_beta"] = list(body_betas) + existing_beta: Final = headers.get("anthropic-beta", "") + header_betas: Final = beta_set | {beta.strip() for beta in existing_beta.split(",") if beta.strip()} + if header_betas: + headers["anthropic-beta"] = ",".join(header_betas) return data diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 8b9729fa9f0..3686b1b017e 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4188,3 +4188,44 @@ def test_anthropic_error_frame_exception_classes_an_overloaded_frame_as_internal error: Final = anthropic_error_frame_exception("overloaded_error", "Overloaded", 503, "claude-sonnet-4-5") assert type(error) is litellm.InternalServerError + + +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} + + +@pytest.mark.parametrize( + "provider,expected", + [("anthropic", True), ("vertex_ai", True), ("azure_ai", False), ("bedrock", False), ("bedrock_converse", False)], +) +@pytest.mark.parametrize( + "optional_params,messages", + [ + ({"compaction": {"type": "summarize"}}, [{"role": "user", "content": "hi"}]), + ({}, [{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "go on"}]), + ], + ids=["compaction_param", "signed_compaction_block"], +) +def test_requires_native_compaction_beta_per_provider( + provider: str, expected: bool, optional_params: dict[str, object], messages: list[object] +) -> None: + from litellm.llms.anthropic.common_utils import requires_native_compaction_beta + + assert requires_native_compaction_beta(provider, optional_params, messages) is expected + + +@pytest.mark.parametrize("provider", ["anthropic", "vertex_ai"]) +@pytest.mark.parametrize( + "optional_params,messages", + [ + ({"max_tokens": 64}, [{"role": "user", "content": "hi"}]), + ({}, [{"role": "assistant", "content": [{"type": "compaction", "content": "unsigned"}]}]), + ({"compaction": None}, [{"role": "user", "content": "hi"}]), + ], + ids=["no_signal", "unsigned_block", "compaction_none"], +) +def test_requires_native_compaction_beta_is_false_without_a_signal( + provider: str, optional_params: dict[str, object], messages: list[object] +) -> None: + from litellm.llms.anthropic.common_utils import requires_native_compaction_beta + + assert requires_native_compaction_beta(provider, optional_params, messages) is False diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 67a32cc82bc..72a97a5f66f 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch import pytest +from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) @@ -126,7 +127,7 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out(): assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "") -def _validate_vertex_headers(client_headers, messages): +def _validate_vertex_headers(client_headers, messages, optional_params=None): config = VertexAIPartnerModelsAnthropicMessagesConfig() litellm_params = { "vertex_ai_project": "test-project", @@ -142,7 +143,7 @@ def _validate_vertex_headers(client_headers, messages): headers=client_headers, model="claude-opus-5-5", messages=messages, - optional_params={"max_tokens": 64}, + optional_params=optional_params or {"max_tokens": 64}, litellm_params=litellm_params, api_base=None, ) @@ -726,3 +727,43 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f and info.get("supports_mid_conversation_system") is not True ] assert missing == [] + + +_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"} + + +@pytest.mark.parametrize( + "client_headers", + [{"anthropic-beta": "compact-2026-09-04"}, {}], + ids=["client_sends_beta", "client_omits_beta"], +) +@pytest.mark.parametrize( + "optional_params,messages", + [ + ({"max_tokens": 64, "compaction": {"type": "summarize"}}, [{"role": "user", "content": "Hello"}]), + ( + {"max_tokens": 64}, + [{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "Go on"}], + ), + ], + ids=["compaction_param", "signed_compaction_block"], +) +def test_native_compaction_reaches_vertex_with_compact_2026_09_04_beta( + local_beta_headers_config, client_headers, optional_params, messages +): + """Vertex serves on-demand compaction only behind compact-2026-09-04 (Google Cloud: beta on + https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) + and answers 400 "compaction: Extra inputs are not permitted" without it, so the beta rides + along with the `compaction` param or a signed compaction block exactly once, whether or not + the client sent it, and survives the Vertex beta filter.""" + headers = _validate_vertex_headers(client_headers, messages, optional_params) + + filtered = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") + + assert filtered["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + + +def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out(): + headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) + + assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index ca4a3dedb4a..ed7da939bdc 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -6,6 +6,7 @@ import pytest from litellm.anthropic_beta_headers_manager import ( update_headers_with_filtered_beta, + update_request_with_filtered_beta, ) from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, @@ -875,3 +876,104 @@ def test_chat_flagged_model_replays_a_byte_identical_prefix_around_a_mid_convers _assert_prefix_stable(requests) assert [m["role"] for m in requests[1]["messages"]] == ["user", "assistant", "user", "system"] assert [m["role"] for m in requests[2]["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"] + + +_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"} + + +@pytest.mark.parametrize( + "optional_params,messages", + [ + ( + {"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}}, + [{"role": "user", "content": "Hello"}], + ), + ( + {"max_tokens": 100, "is_vertex_request": True}, + [ + { + "role": "assistant", + "content": "", + "provider_specific_fields": {"compaction_blocks": [_SIGNED_COMPACTION_BLOCK]}, + }, + {"role": "user", "content": "Go on"}, + ], + ), + ], + ids=["compaction_param", "signed_compaction_block_sent_back"], +) +def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta( + local_beta_headers_config, optional_params, messages +): + """The chat path skips the Anthropic beta injection for Vertex requests, so the Vertex config + adds compact-2026-09-04 itself (Google Cloud: beta on + https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) + as the HTTP header only, since Vertex answered 400 "Unexpected value(s) `compact-2026-09-04` + for the `anthropic_beta` parameter" with the name in the body field (us-east5, + claude-sonnet-4-6, 2026-10-05), and the header must survive the Vertex beta filter.""" + config = VertexAIAnthropicConfig() + headers = {} + + result = config.transform_request( + model="claude-opus-4-6", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + filtered_headers, filtered_body = update_request_with_filtered_beta( + headers=headers, request_data=result, provider="vertex_ai" + ) + + assert filtered_headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert "compact-2026-09-04" not in filtered_body.get("anthropic_beta", []) + + +def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out(): + headers = {} + + result = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"max_tokens": 100, "is_vertex_request": True}, + litellm_params={}, + headers=headers, + ) + + assert "compact-2026-09-04" not in result.get("anthropic_beta", []) + assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") + + +def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(local_beta_headers_config): + headers = {} + + result = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={ + "max_tokens": 100, + "is_vertex_request": True, + "compaction": {"type": "summarize"}, + "extra_headers": {"anthropic-beta": "compact-2026-09-04,interleaved-thinking-2025-05-14"}, + }, + litellm_params={}, + headers=headers, + ) + + assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] + + +def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(local_beta_headers_config): + headers = {"anthropic-beta": "context-1m-2025-08-07"} + + result = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}}, + litellm_params={}, + headers=headers, + ) + + assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"] + assert "anthropic_beta" not in result diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index 1a6899f16ba..e4bf891ad80 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -83,7 +83,7 @@ class TestAnthropicBetaHeadersFiltering: filtered = filter_and_transform_beta_headers( beta_headers=all_headers, provider=provider ) - assert ("compact-2026-09-04" in filtered) is (provider == "anthropic") + assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "vertex_ai")) for header in unsupported_headers: assert ( @@ -405,14 +405,16 @@ class TestAnthropicBetaHeadersFiltering: ), f"Header '{test_case['input']}' should be mapped to '{test_case['expected']}' for {test_case['provider']}, but got: {filtered}" def test_filter_and_transform_beta_headers_vertex_ai_keeps_compact(self): - """Vertex AI supports compact context edits, so the compact beta header - must be forwarded instead of stripped (it was previously mapped to null, - which broke compact_20260112 context edits over /v1/messages).""" + """Vertex AI serves both compaction betas (threshold compaction behind compact-2026-01-12 + and on-demand compaction behind compact-2026-09-04, Google Cloud: beta on + https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05), + so the Vertex filter forwards both instead of stripping them: a dropped header makes + Vertex answer 400 "compaction: Extra inputs are not permitted".""" filtered = filter_and_transform_beta_headers( - beta_headers=["compact-2026-01-12"], provider="vertex_ai" + beta_headers=["compact-2026-01-12", "compact-2026-09-04"], provider="vertex_ai" ) - assert filtered == ["compact-2026-01-12"] + assert filtered == ["compact-2026-01-12", "compact-2026-09-04"] @pytest.mark.parametrize("provider", ["bedrock_converse", "bedrock"]) def test_fine_grained_tool_streaming_forwarded_for_bedrock(self, provider): From a964e843012724c42f5b31b2232e6e962e14fc72 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:32:29 -0700 Subject: [PATCH 02/12] test(vertex_ai): prove the compact-2026-09-04 beta reaches the wire through both Claude handlers --- ...artner_models_anthropic_messages_config.py | 51 ++++++++++++++++++- ...partner_models_anthropic_transformation.py | 48 +++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 72a97a5f66f..7fc40508266 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -1,14 +1,18 @@ import copy import json import os -from unittest.mock import MagicMock, patch +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import litellm from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels from litellm.types.router import GenericLiteLLMParams @@ -767,3 +771,48 @@ def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out(): headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") + + +_VERTEX_MESSAGE_RESPONSE: Final = { + "id": "msg_vrtx_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "pong"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 1}, +} + + +@pytest.mark.asyncio +async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config): + """A /v1/messages compaction request to a Vertex Claude deployment reaches Vertex with the beta in the + anthropic-beta request header and never in the body, the shape Vertex accepts (live, 2026-10-05).""" + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.text = json.dumps(_VERTEX_MESSAGE_RESPONSE) + mock_response.headers = httpx.Headers({}) + mock_response.json.return_value = _VERTEX_MESSAGE_RESPONSE + + with ( + patch.object(AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_response) as mock_post, + patch( + "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token", + return_value=("token", "test-project"), + ), + ): + await litellm.anthropic.messages.acreate( + model="vertex_ai/claude-sonnet-4-6", + max_tokens=64, + messages=[{"role": "user", "content": "Reply with the single word: pong"}], + compaction={"type": "summarize"}, + vertex_project="test-project", + vertex_location="us-east5", + ) + + sent: Final = mock_post.call_args.kwargs + sent_body: Final = sent["json"] if "json" in sent else json.loads(sent["data"]) + assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert sent_body["compaction"] == {"type": "summarize"} + assert "anthropic_beta" not in sent_body diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index ed7da939bdc..742fe27cd8d 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1,9 +1,14 @@ import copy import json +from typing import Final +from unittest.mock import MagicMock, patch import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler + from litellm.anthropic_beta_headers_manager import ( update_headers_with_filtered_beta, update_request_with_filtered_beta, @@ -977,3 +982,46 @@ def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_heade assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"] assert "anthropic_beta" not in result + + +_VERTEX_MESSAGE_RESPONSE: Final = { + "id": "msg_vrtx_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "pong"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 1}, +} + + +def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config): + """A chat completion with `compaction` on a Vertex Claude deployment reaches Vertex with the beta in the + anthropic-beta request header and never in the body field, the shape Vertex accepts (live, 2026-10-05).""" + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.text = json.dumps(_VERTEX_MESSAGE_RESPONSE) + mock_response.headers = {} + mock_response.json.return_value = _VERTEX_MESSAGE_RESPONSE + client: Final = MagicMock(spec=HTTPHandler) + client.post.return_value = mock_response + + with patch( + "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token", + return_value=("token", "test-project"), + ): + litellm.completion( + model="vertex_ai/claude-sonnet-4-6", + messages=[{"role": "user", "content": "Reply with the single word: pong"}], + compaction={"type": "summarize"}, + vertex_project="test-project", + vertex_location="us-east5", + client=client, + ) + + sent: Final = client.post.call_args.kwargs + sent_body: Final = sent["json"] if "json" in sent else json.loads(sent["data"]) + assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert sent_body["compaction"] == {"type": "summarize"} + assert "anthropic_beta" not in sent_body From e189fe66b9582fb805bae805d091767b0b04d084 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:47:25 -0700 Subject: [PATCH 03/12] fix(vertex_ai): move a client-sent body compact-2026-09-04 beta to the request header --- .../anthropic/transformation.py | 19 +++++++-------- ...partner_models_anthropic_transformation.py | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 7f9e0a0ea3e..2e6da9677e3 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -21,6 +21,14 @@ if TYPE_CHECKING: _VERTEX_HEADER_ONLY_BETAS: Final = frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) +def _beta_names(value: object) -> frozenset[str]: + if isinstance(value, str): + return frozenset(beta.strip() for beta in value.split(",") if beta.strip()) + if isinstance(value, list): + return frozenset(str(beta).strip() for beta in value if str(beta).strip()) + return frozenset() + + class VertexAIError(Exception): def __init__(self, status_code, message): self.status_code = status_code @@ -139,16 +147,9 @@ class VertexAIAnthropicConfig(AnthropicConfig): beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) extra_headers: Final = optional_params.get("extra_headers") or {} - anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "") - if isinstance(anthropic_beta_value, str) and anthropic_beta_value: - for beta in anthropic_beta_value.split(","): - beta = beta.strip() - if beta: - beta_set.add(beta) - elif isinstance(anthropic_beta_value, list): - beta_set.update(anthropic_beta_value) - + beta_set.update(_beta_names(extra_headers.get("anthropic-beta"))) data.pop("extra_headers", None) + beta_set.update(_beta_names(data.pop("anthropic_beta", None))) body_betas: Final = beta_set - _VERTEX_HEADER_ONLY_BETAS if body_betas: diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 742fe27cd8d..6ef8782c181 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -984,6 +984,29 @@ def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_heade assert "anthropic_beta" not in result +def test_vertex_ai_anthropic_client_sent_body_compact_2026_09_04_moves_to_the_header(local_beta_headers_config): + """Vertex rejects `compact-2026-09-04` inside the body `anthropic_beta` field with 400 "Unexpected value(s)" + (live, 2026-10-05), so a client that opts in through that field still gets the beta as a header and keeps + its other betas in the body.""" + headers = {} + + result = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={ + "max_tokens": 100, + "is_vertex_request": True, + "compaction": {"type": "summarize"}, + "anthropic_beta": ["compact-2026-09-04", "interleaved-thinking-2025-05-14"], + }, + litellm_params={}, + headers=headers, + ) + + assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] + + _VERTEX_MESSAGE_RESPONSE: Final = { "id": "msg_vrtx_test", "type": "message", From 3b3ea1ea031b685b3bc95d83e9dab88f9bb4ffb9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:11:09 -0700 Subject: [PATCH 04/12] refactor(vertex_ai): build the Claude beta set without mutation and inject the client in the messages wire test --- .../anthropic/transformation.py | 21 ++++++++++++------- ...artner_models_anthropic_messages_config.py | 15 ++++++------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 2e6da9677e3..b7b56d68628 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -143,19 +143,24 @@ class VertexAIAnthropicConfig(AnthropicConfig): if context_management: self._add_context_management_beta_headers(beta_set, context_management) - if requires_native_compaction_beta("vertex_ai", optional_params, data["messages"]): - beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) - + compaction_betas: Final = ( + frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) + if requires_native_compaction_beta("vertex_ai", optional_params, data["messages"]) + else frozenset() + ) extra_headers: Final = optional_params.get("extra_headers") or {} - beta_set.update(_beta_names(extra_headers.get("anthropic-beta"))) data.pop("extra_headers", None) - beta_set.update(_beta_names(data.pop("anthropic_beta", None))) + requested_betas: Final = ( + frozenset(beta_set) + | compaction_betas + | _beta_names(extra_headers.get("anthropic-beta")) + | _beta_names(data.pop("anthropic_beta", None)) + ) - body_betas: Final = beta_set - _VERTEX_HEADER_ONLY_BETAS + body_betas: Final = requested_betas - _VERTEX_HEADER_ONLY_BETAS if body_betas: data["anthropic_beta"] = list(body_betas) - existing_beta: Final = headers.get("anthropic-beta", "") - header_betas: Final = beta_set | {beta.strip() for beta in existing_beta.split(",") if beta.strip()} + header_betas: Final = requested_betas | _beta_names(headers.get("anthropic-beta")) if header_betas: headers["anthropic-beta"] = ",".join(header_betas) diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 7fc40508266..85c5fc43946 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -795,12 +795,12 @@ async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire mock_response.headers = httpx.Headers({}) mock_response.json.return_value = _VERTEX_MESSAGE_RESPONSE - with ( - patch.object(AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_response) as mock_post, - patch( - "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token", - return_value=("token", "test-project"), - ), + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock(return_value=mock_response) + + with patch( + "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token", + return_value=("token", "test-project"), ): await litellm.anthropic.messages.acreate( model="vertex_ai/claude-sonnet-4-6", @@ -809,9 +809,10 @@ async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire compaction={"type": "summarize"}, vertex_project="test-project", vertex_location="us-east5", + client=client, ) - sent: Final = mock_post.call_args.kwargs + sent: Final = client.post.call_args.kwargs sent_body: Final = sent["json"] if "json" in sent else json.loads(sent["data"]) assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 assert sent_body["compaction"] == {"type": "summarize"} From b94eb3ec880cf0e6b6b8cce9cce54573a0c61a36 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:39:34 -0700 Subject: [PATCH 05/12] fix(vertex_ai): keep client body betas in the body and answer 400 on a malformed anthropic_beta --- .../anthropic/transformation.py | 34 ++++++++++------- ...partner_models_anthropic_transformation.py | 37 ++++++++++++++++++- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index b7b56d68628..bcceda2acfd 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -12,7 +12,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from ....anthropic.chat.transformation import AnthropicConfig -from ....anthropic.common_utils import requires_native_compaction_beta +from ....anthropic.common_utils import AnthropicError, requires_native_compaction_beta from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: @@ -22,11 +22,16 @@ _VERTEX_HEADER_ONLY_BETAS: Final = frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPA def _beta_names(value: object) -> frozenset[str]: + if value is None: + return frozenset() if isinstance(value, str): return frozenset(beta.strip() for beta in value.split(",") if beta.strip()) - if isinstance(value, list): - return frozenset(str(beta).strip() for beta in value if str(beta).strip()) - return frozenset() + if isinstance(value, (list, tuple)) and all(isinstance(beta, str) for beta in value): + return frozenset(beta.strip() for beta in value if beta.strip()) + raise AnthropicError( + status_code=400, + message=f"anthropic_beta must be a string or a list of strings, got {type(value).__name__}", + ) class VertexAIError(Exception): @@ -150,19 +155,20 @@ class VertexAIAnthropicConfig(AnthropicConfig): ) extra_headers: Final = optional_params.get("extra_headers") or {} data.pop("extra_headers", None) - requested_betas: Final = ( - frozenset(beta_set) - | compaction_betas - | _beta_names(extra_headers.get("anthropic-beta")) - | _beta_names(data.pop("anthropic_beta", None)) - ) + injected_betas: Final = frozenset(beta_set) | _beta_names(extra_headers.get("anthropic-beta")) + client_body_betas: Final = _beta_names(data.pop("anthropic_beta", None)) - body_betas: Final = requested_betas - _VERTEX_HEADER_ONLY_BETAS + body_betas: Final = (injected_betas | client_body_betas) - _VERTEX_HEADER_ONLY_BETAS if body_betas: - data["anthropic_beta"] = list(body_betas) - header_betas: Final = requested_betas | _beta_names(headers.get("anthropic-beta")) + data["anthropic_beta"] = sorted(body_betas) + header_betas: Final = ( + injected_betas + | compaction_betas + | (client_body_betas & _VERTEX_HEADER_ONLY_BETAS) + | _beta_names(headers.get("anthropic-beta")) + ) if header_betas: - headers["anthropic-beta"] = ",".join(header_betas) + headers["anthropic-beta"] = ",".join(sorted(header_betas)) return data diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 6ef8782c181..b55d63e400a 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -13,6 +13,7 @@ from litellm.anthropic_beta_headers_manager import ( update_headers_with_filtered_beta, update_request_with_filtered_beta, ) +from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, ) @@ -1003,10 +1004,44 @@ def test_vertex_ai_anthropic_client_sent_body_compact_2026_09_04_moves_to_the_he headers=headers, ) - assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert headers["anthropic-beta"] == "compact-2026-09-04" assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] +@pytest.mark.parametrize( + "anthropic_beta", + [["interleaved-thinking-2025-05-14"], ("interleaved-thinking-2025-05-14",)], + ids=["list", "sdk_tuple"], +) +def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta): + headers = {} + + result = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta}, + litellm_params={}, + headers=headers, + ) + + assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] + assert "anthropic-beta" not in headers + + +@pytest.mark.parametrize("anthropic_beta", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"]) +def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400(anthropic_beta): + with pytest.raises(AnthropicError) as excinfo: + VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta}, + litellm_params={}, + headers={}, + ) + + assert excinfo.value.status_code == 400 + + _VERTEX_MESSAGE_RESPONSE: Final = { "id": "msg_vrtx_test", "type": "message", From c4862e8b49dbf3428f507317686fa827a0da5ded Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:41:14 -0700 Subject: [PATCH 06/12] fix(rust): allow the compact-2026-09-04 beta on vertex_ai in the Rust beta table The beta header config now maps compact-2026-09-04 for vertex_ai, so the Rust table that mirrors it must stop rejecting that beta there, and the Rust Vertex messages route adds it for a compaction request like the Python route does. --- .../crates/llms-types/src/providers/anthropic/beta.rs | 9 ++++++++- .../crates/llms/src/vertex_ai/messages/transformation.rs | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index b1509313b88..779c5f9d00e 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -199,7 +199,7 @@ impl AnthropicBeta { &[BetaProvider::BedrockConverse, BetaProvider::Databricks] } Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse], - Self::AdvisorTool20260301 | Self::Compact20260904 => &[ + Self::AdvisorTool20260301 => &[ BetaProvider::AzureAi, BetaProvider::BedrockConverse, BetaProvider::Bedrock, @@ -207,6 +207,13 @@ impl AnthropicBeta { BetaProvider::VertexAi, BetaProvider::Databricks, ], + Self::Compact20260904 => &[ + BetaProvider::AzureAi, + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::BedrockMantle, + BetaProvider::Databricks, + ], Self::Bash20241022 | Self::Bash20250124 | Self::McpServers20251204 diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 2e77dbc1ca8..f4e8ab3bcca 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -661,6 +661,7 @@ mod tests { ]}}), &["compact-2026-01-12", "context-management-2025-06-27"] )] + #[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])] #[case::advisor_tool_is_not_a_vertex_beta( json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}), &[] From cdc280c86712662607a00ee1da53add5853cf5bd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:46:31 -0700 Subject: [PATCH 07/12] fix(vertex_ai): keep client body betas out of the Vertex body next to litellm betas, honor drop_params, and send the compaction beta on count_tokens Litellm's own betas replace a client's body anthropic_beta again, as on main, so a body beta Vertex rejects no longer turns a 200 into a 400. A malformed beta drops under drop_params and names its field otherwise. inline-tools-2026-09-15 joins compact-2026-09-04 as header-only, since Vertex rejects both in the body. Vertex count_tokens sends compact-2026-09-04 when the request carries compaction. --- .../anthropic/transformation.py | 42 ++-- .../count_tokens/handler.py | 20 +- litellm/types/llms/anthropic.py | 1 + ...artner_models_anthropic_messages_config.py | 23 ++- ...partner_models_anthropic_transformation.py | 181 +++++++++++------- .../count_tokens/test_handler.py | 34 ++++ 6 files changed, 206 insertions(+), 95 deletions(-) create mode 100644 tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 692a35f3a1f..18022f8d51d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -22,7 +22,9 @@ if TYPE_CHECKING: _NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object]) -_VERTEX_HEADER_ONLY_BETAS: Final = frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) +_VERTEX_HEADER_ONLY_BETAS: Final = frozenset( + {ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value, ANTHROPIC_BETA_HEADER_VALUES.INLINE_TOOLS_2026_09_15.value} +) _BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None) @@ -30,18 +32,27 @@ _BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str _REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object]) -def _beta_names(value: object) -> frozenset[str]: +def _beta_names(value: object) -> frozenset[str] | None: try: parsed: Final = _BETA_VALUE.validate_python(value) - except ValidationError as error: - raise AnthropicError( - status_code=400, - message=f"anthropic_beta must be a string or a list of strings, got {type(value).__name__}", - ) from error + except ValidationError: + return None names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or () return frozenset(name.strip() for name in names if name.strip()) +def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]: + names: Final = _beta_names(value) + if names is not None: + return names + if drop_params: + return frozenset() + raise AnthropicError( + status_code=400, + message=f"{field} must be a string or a list of strings, got {type(value).__name__}", + ) + + class VertexAIError(Exception): def __init__(self, status_code, message): self.status_code = status_code @@ -162,12 +173,17 @@ class VertexAIAnthropicConfig(AnthropicConfig): if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"])) else frozenset[str]() ) + drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {}) data.pop("extra_headers", None) - injected_betas: Final = frozenset(beta_set) | _beta_names(extra_headers.get("anthropic-beta")) - client_body_betas: Final = _beta_names(_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta")) + injected_betas: Final = frozenset(beta_set) | _client_betas( + extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params + ) + client_body_betas: Final = _client_betas( + _NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params + ) - body_betas: Final = (injected_betas | client_body_betas) - _VERTEX_HEADER_ONLY_BETAS + body_betas: Final = (injected_betas or client_body_betas) - _VERTEX_HEADER_ONLY_BETAS if body_betas: data["anthropic_beta"] = sorted(body_betas) elif "anthropic_beta" in data: @@ -176,7 +192,11 @@ class VertexAIAnthropicConfig(AnthropicConfig): injected_betas | compaction_betas | (client_body_betas & _VERTEX_HEADER_ONLY_BETAS) - | _beta_names(_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta")) + | _client_betas( + _NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"), + "The anthropic-beta header", + drop_params, + ) ) if header_betas: headers["anthropic-beta"] = ",".join(sorted(header_betas)) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 7100c9efde3..103c51eb71a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,11 +6,16 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Mapping, Sequence from typing import Any, Final +from pydantic import TypeAdapter + +from litellm.llms.anthropic.common_utils import requires_native_compaction_beta from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES _COUNT_TOKENS_PUBLISHERS: Final = ( ("claude", "anthropic"), @@ -19,10 +24,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( ) +_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object]) + + def count_tokens_publisher(model: str) -> str | None: return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None) +def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]: + authorization: Final = {"Authorization": f"Bearer {access_token}"} + if not requires_native_compaction_beta( + "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"]) + ): + return authorization + return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value} + + class VertexAIPartnerModelsTokenCounter(VertexBase): """ Token counter for Vertex AI Partner Models. @@ -157,8 +174,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): api_base=litellm_params.get("api_base"), ) - # Prepare headers - headers: Final = {"Authorization": f"Bearer {access_token}"} + headers: Final = dict(count_tokens_headers(access_token, request_data)) # Get async HTTP client from litellm import LlmProviders diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 6aeead7679f..9961ae12852 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -794,6 +794,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" COMPACT_2026_01_12 = "compact-2026-01-12" COMPACT_2026_09_04 = "compact-2026-09-04" + INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 6d3d1639615..3bf73debf65 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -132,7 +132,9 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out(): assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "") -def _validate_vertex_headers(client_headers, messages, optional_params=None): +def _validate_vertex_headers( + client_headers: dict[str, str], messages: list[object], optional_params: dict[str, object] | None = None +) -> dict[str, str]: config = VertexAIPartnerModelsAnthropicMessagesConfig() litellm_params = { "vertex_ai_project": "test-project", @@ -829,7 +831,7 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f assert missing == [] -_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"} +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} @pytest.mark.parametrize( @@ -849,22 +851,25 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", " ids=["compaction_param", "signed_compaction_block"], ) def test_native_compaction_reaches_vertex_with_compact_2026_09_04_beta( - local_beta_headers_config, client_headers, optional_params, messages -): + local_beta_headers_config: None, + client_headers: dict[str, str], + optional_params: dict[str, object], + messages: list[object], +) -> None: """Vertex serves on-demand compaction only behind compact-2026-09-04 (Google Cloud: beta on https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) and answers 400 "compaction: Extra inputs are not permitted" without it, so the beta rides along with the `compaction` param or a signed compaction block exactly once, whether or not the client sent it, and survives the Vertex beta filter.""" - headers = _validate_vertex_headers(client_headers, messages, optional_params) + headers: Final = _validate_vertex_headers(client_headers, messages, optional_params) - filtered = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") + filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") assert filtered["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 -def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out(): - headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) +def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None: + headers: Final = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") @@ -882,7 +887,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = { @pytest.mark.asyncio -async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config): +async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None: """A /v1/messages compaction request to a Vertex Claude deployment reaches Vertex with the beta in the anthropic-beta request header and never in the body, the shape Vertex accepts (live, 2026-10-05).""" mock_response: Final = MagicMock() diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index d90bb944116..67a74d0fb4f 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -430,8 +430,10 @@ def test_vertex_ai_anthropic_extra_headers_beta_propagation(): def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(local_beta_headers_config: None) -> None: """The chat path runs the Vertex beta filter over both the header and the `anthropic_beta` body field right - before the request goes out, so inline-tools-2026-09-15 must survive both (source and date in - tests/unit/test_anthropic_beta_headers_filtering.py).""" + before the request goes out, so inline-tools-2026-09-15 must survive it as the header (source and date in + tests/unit/test_anthropic_beta_headers_filtering.py) and stay out of the body field, which Vertex rejects + with 400 "Unexpected value(s) `inline-tools-2026-09-15` for the `anthropic_beta` parameter" (live, global, + claude-sonnet-5-5, 2026-10-20).""" config: Final = VertexAIAnthropicConfig() headers: Final[dict[str, str]] = {} optional_params: Final[dict[str, object]] = { @@ -452,7 +454,7 @@ def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(loc ) assert filtered_headers["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1 - assert "inline-tools-2026-09-15" in filtered_request["anthropic_beta"] + assert "inline-tools-2026-09-15" not in filtered_request.get("anthropic_beta", []) def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas(): @@ -1189,7 +1191,18 @@ def test_refusal_stop_details_kept_through_vertex_anthropic_transform(): assert provider_fields["stop_details"] == ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"] -_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"} +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} +_CLEAR_TOOL_USES: Final = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + +def _vertex_request(optional_params: dict[str, object], headers: dict[str, str], litellm_params: dict[str, object] | None = None) -> dict: + return VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"max_tokens": 100, "is_vertex_request": True, **optional_params}, + litellm_params=litellm_params or {}, + headers=headers, + ) @pytest.mark.parametrize( @@ -1214,18 +1227,17 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", " ids=["compaction_param", "signed_compaction_block_sent_back"], ) def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta( - local_beta_headers_config, optional_params, messages -): + local_beta_headers_config: None, optional_params: dict[str, object], messages: list[dict[str, object]] +) -> None: """The chat path skips the Anthropic beta injection for Vertex requests, so the Vertex config adds compact-2026-09-04 itself (Google Cloud: beta on https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) as the HTTP header only, since Vertex answered 400 "Unexpected value(s) `compact-2026-09-04` for the `anthropic_beta` parameter" with the name in the body field (us-east5, claude-sonnet-4-6, 2026-10-05), and the header must survive the Vertex beta filter.""" - config = VertexAIAnthropicConfig() - headers = {} + headers: Final[dict[str, str]] = {} - result = config.transform_request( + result: Final = VertexAIAnthropicConfig().transform_request( model="claude-opus-4-6", messages=messages, optional_params=optional_params, @@ -1240,111 +1252,134 @@ def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta( assert "compact-2026-09-04" not in filtered_body.get("anthropic_beta", []) -def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out(): - headers = {} +def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None: + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True}, - litellm_params={}, - headers=headers, - ) + result: Final = _vertex_request({}, headers) assert "compact-2026-09-04" not in result.get("anthropic_beta", []) assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") -def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(local_beta_headers_config): - headers = {} +def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field( + local_beta_headers_config: None, +) -> None: + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={ - "max_tokens": 100, - "is_vertex_request": True, + result: Final = _vertex_request( + { "compaction": {"type": "summarize"}, "extra_headers": {"anthropic-beta": "compact-2026-09-04,interleaved-thinking-2025-05-14"}, }, - litellm_params={}, - headers=headers, + headers, ) assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] -def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(local_beta_headers_config): - headers = {"anthropic-beta": "context-1m-2025-08-07"} +def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers( + local_beta_headers_config: None, +) -> None: + headers: Final = {"anthropic-beta": "context-1m-2025-08-07"} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}}, - litellm_params={}, - headers=headers, - ) + result: Final = _vertex_request({"compaction": {"type": "summarize"}}, headers) assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"] assert "anthropic_beta" not in result -def test_vertex_ai_anthropic_client_sent_body_compact_2026_09_04_moves_to_the_header(local_beta_headers_config): - """Vertex rejects `compact-2026-09-04` inside the body `anthropic_beta` field with 400 "Unexpected value(s)" - (live, 2026-10-05), so a client that opts in through that field still gets the beta as a header and keeps - its other betas in the body.""" - headers = {} +@pytest.mark.parametrize("header_only_beta", ["compact-2026-09-04", "inline-tools-2026-09-15"]) +def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_header( + local_beta_headers_config: None, header_only_beta: str +) -> None: + """Vertex rejects these betas inside the body `anthropic_beta` field with 400 "Unexpected value(s)" and + accepts them as the header (live, global, claude-sonnet-5-5, 2026-10-20), so a client that opts in through that field still + gets the beta as a header and keeps its other betas in the body.""" + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={ - "max_tokens": 100, - "is_vertex_request": True, - "compaction": {"type": "summarize"}, - "anthropic_beta": ["compact-2026-09-04", "interleaved-thinking-2025-05-14"], - }, - litellm_params={}, - headers=headers, + result: Final = _vertex_request( + {"anthropic_beta": [header_only_beta, "interleaved-thinking-2025-05-14"]}, headers ) - assert headers["anthropic-beta"] == "compact-2026-09-04" + assert headers["anthropic-beta"] == header_only_beta assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] +@pytest.mark.parametrize( + "client_beta,header", + [ + ("context-1m-2025-08-07", "context-management-2025-06-27"), + ("inline-tools-2026-09-15", "context-management-2025-06-27,inline-tools-2026-09-15"), + ], + ids=["body_beta", "header_only_beta"], +) +def test_vertex_ai_anthropic_litellm_betas_replace_the_client_body_betas( + local_beta_headers_config: None, client_beta: str, header: str +) -> None: + headers: Final[dict[str, str]] = {} + + result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers) + + assert result["anthropic_beta"] == ["context-management-2025-06-27"] + assert headers["anthropic-beta"] == header + + @pytest.mark.parametrize( "anthropic_beta", [["interleaved-thinking-2025-05-14"], ("interleaved-thinking-2025-05-14",)], ids=["list", "sdk_tuple"], ) -def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta): - headers = {} +def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta: object) -> None: + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta}, - litellm_params={}, - headers=headers, - ) + result: Final = _vertex_request({"anthropic_beta": anthropic_beta}, headers) assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] assert "anthropic-beta" not in headers -@pytest.mark.parametrize("anthropic_beta", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"]) -def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400(anthropic_beta): +_MALFORMED_BETAS: Final = pytest.mark.parametrize( + "malformed", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"] +) +_MALFORMED_BETA_SOURCES: Final = pytest.mark.parametrize( + "source,field", + [("anthropic_beta", "anthropic_beta"), ("extra_headers", "The anthropic-beta header")], + ids=["body_field", "extra_header"], +) + + +def _malformed_beta_params(source: str, malformed: object) -> dict[str, object]: + if source == "extra_headers": + return {"extra_headers": {"anthropic-beta": malformed}} + return {"anthropic_beta": malformed} + + +@_MALFORMED_BETAS +@_MALFORMED_BETA_SOURCES +def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400_naming_its_field( + malformed: object, source: str, field: str +) -> None: with pytest.raises(AnthropicError) as excinfo: - VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta}, - litellm_params={}, - headers={}, - ) + _vertex_request(_malformed_beta_params(source, malformed), {}) assert excinfo.value.status_code == 400 + assert excinfo.value.message.startswith(f"{field} must be") + + +@_MALFORMED_BETAS +@_MALFORMED_BETA_SOURCES +@pytest.mark.parametrize("auto_beta", [False, True], ids=["alone", "next_to_litellm_beta"]) +def test_vertex_ai_anthropic_malformed_anthropic_beta_is_dropped_under_drop_params( + local_beta_headers_config: None, malformed: object, source: str, field: str, auto_beta: bool +) -> None: + headers: Final[dict[str, str]] = {} + params: Final = {**_malformed_beta_params(source, malformed), **({"context_management": _CLEAR_TOOL_USES} if auto_beta else {})} + + result: Final = _vertex_request(params, headers, litellm_params={"drop_params": True}) + + assert result.get("anthropic_beta") == (["context-management-2025-06-27"] if auto_beta else None) + assert headers.get("anthropic-beta") == ("context-management-2025-06-27" if auto_beta else None) _VERTEX_MESSAGE_RESPONSE: Final = { @@ -1359,7 +1394,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = { } -def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config): +def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None: """A chat completion with `compaction` on a Vertex Claude deployment reaches Vertex with the beta in the anthropic-beta request header and never in the body field, the shape Vertex accepts (live, 2026-10-05).""" mock_response: Final = MagicMock() diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py new file mode 100644 index 00000000000..b23461823f8 --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py @@ -0,0 +1,34 @@ +from typing import Final + +import pytest + +from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_headers + +_HISTORY: Final = ({"role": "user", "content": "Hello"},) + + +@pytest.mark.parametrize( + "request_data", + [ + {"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": "sig"}]}]}, + {"messages": list(_HISTORY), "compaction": {"type": "summarize"}}, + ], + ids=["signed_compaction_block", "compaction_param"], +) +def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_data: dict[str, object]) -> None: + assert count_tokens_headers("token", request_data) == { + "Authorization": "Bearer token", + "anthropic-beta": "compact-2026-09-04", + } + + +@pytest.mark.parametrize( + "request_data", + [ + {"messages": list(_HISTORY)}, + {"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": ""}]}]}, + ], + ids=["plain", "unsigned_compaction_block"], +) +def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None: + assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"} From c8097f9c378dba72da554f4d591432c2a24a0b2e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:34:05 -0700 Subject: [PATCH 08/12] fix(logging): log and bill Anthropic messages compaction responses on the Rust route The Rust route hands the served Anthropic body to the logger, which validates it against AnthropicResponse. That model had no compaction content block and no compaction stop reason, so a compaction reply raised and wrote no spend row. --- litellm/types/llms/anthropic.py | 11 +- .../test_litellm_logging.py | 259 +++++++++++------- 2 files changed, 171 insertions(+), 99 deletions(-) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 9961ae12852..55c383a25a7 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -698,6 +698,14 @@ class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel): data: str +class AnthropicResponseContentBlockCompaction(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["compaction"] + content: str | None + signature: str | None = None + + class AnthropicResponseUsageBlock(LiteLLMBaseModel): model_config = ConfigDict(extra="allow") @@ -711,7 +719,7 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel): thinking_tokens: int | None = None -AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] +AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] class AnthropicResponse(LiteLLMBaseModel): @@ -729,6 +737,7 @@ class AnthropicResponse(LiteLLMBaseModel): | AnthropicResponseContentBlockToolUse | AnthropicResponseContentBlockThinking | AnthropicResponseContentBlockRedactedThinking + | AnthropicResponseContentBlockCompaction ] """Content generated by the model.""" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 9d651b4ae3e..608b95bbaeb 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -75,6 +75,7 @@ import inspect from typing import List, Optional from litellm.types.utils import LiteLLMCommonStrings + @pytest.fixture def logging_obj(): return LitellmLogging( @@ -345,8 +346,10 @@ async def test_mcp_native_structured_replacement_must_match_returned_content( ) logging_obj.dynamic_success_callbacks = [NativeReplacement()] returned = await logging_obj.async_post_mcp_tool_call_hook( - kwargs={"original_response": result}, response_obj=result, - start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + kwargs={"original_response": result}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), ) assert returned is result assert result.content == [TextContent(type="text", text="native-safe" if same_content else "final-safe")] @@ -355,7 +358,6 @@ async def test_mcp_native_structured_replacement_must_match_returned_content( assert "SECRET-1234" not in result.model_dump_json() - @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["none", "wrapper", "exception"]) @pytest.mark.parametrize("structured", [False, True]) @@ -373,8 +375,10 @@ async def test_mcp_direct_content_edit_invalidates_stale_structured_data(logging ) logging_obj.dynamic_success_callbacks = [DirectRedactor()] returned = await logging_obj.async_post_mcp_tool_call_hook( - kwargs={"original_response": result}, response_obj=result, - start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + kwargs={"original_response": result}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), ) assert returned is result assert result.content == [TextContent(type="text", text="[REDACTED]")] @@ -2539,7 +2543,6 @@ async def test_completed_success_selects_one_implemented_hook( assert (thread_id == loop_thread) is (kind != "sync") - @pytest.mark.parametrize("for_async_request", [False, True]) @pytest.mark.parametrize("private_first", [False, True]) @pytest.mark.parametrize("response_kind", ["responses", "completion"]) @@ -3227,8 +3230,13 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g class RecordingShadowLogger(ShadowEvalLogger): async def async_log_success_event( - self, kwargs: Mapping[str, object], response_obj: object, start_time: object, - end_time: object, *, guardrail_snapshot: GuardrailRequestSnapshot | None = None, + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + *, + guardrail_snapshot: GuardrailRequestSnapshot | None = None, ) -> None: shadow_snapshots.append(guardrail_snapshot) await super().async_log_success_event( @@ -3237,13 +3245,20 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g class RecordingLogger(CustomLogger): async def async_log_success_event( - self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object, + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, ) -> None: other_payloads.append(kwargs) class LoggingGuardrail(CustomGuardrail): async def async_logging_hook( - self, kwargs: dict[str, object], result: object, call_type: str, + self, + kwargs: dict[str, object], + result: object, + call_type: str, ) -> tuple[dict[str, object], object]: hook_snapshots.append(logging_obj.shadow_eval_request_snapshot) if hook_mode == "raises": @@ -3255,26 +3270,36 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g "user_api_key_hash": "test-key", } snapshot: Final = GuardrailRequestSnapshot.capture( - {"messages": [{"role": "user", "content": "snapshot-only"}]}, metadata, + {"messages": [{"role": "user", "content": "snapshot-only"}]}, + metadata, ) assert snapshot is not None shadow: Final = RecordingShadowLogger(prisma_provider=no_prisma, jobs_cache=InMemoryCache()) guardrail: Final = LoggingGuardrail( - guardrail_name="late-mask", default_on=True, + guardrail_name="late-mask", + default_on=True, event_hook=GuardrailEventHooks.pre_call if hook_mode == "disabled" else GuardrailEventHooks.logging_only, ) monkeypatch.setattr(litellm, "_async_success_callback", []) logging_obj: Final = LitellmLogging( - model="test-model", messages=[], stream=stream, call_type="anthropic_messages", - start_time=datetime.datetime.now(), litellm_call_id="private-snapshot", function_id="private-snapshot", + model="test-model", + messages=[], + stream=stream, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="private-snapshot", + function_id="private-snapshot", dynamic_async_success_callbacks=[shadow, RecordingLogger(), guardrail], ) logging_obj.update_messages([{"role": "user", "content": "logged input"}]) logging_obj.update_environment_variables(litellm_params={"metadata": metadata}, optional_params={}) logging_obj.shadow_eval_request_snapshot = snapshot payload: Final = { - "id": "private-snapshot", "call_type": "anthropic_messages", "metadata": metadata, - "model_group": "test-model", "model_parameters": {}, + "id": "private-snapshot", + "call_type": "anthropic_messages", + "metadata": metadata, + "model_group": "test-model", + "model_parameters": {}, } await logging_obj.async_success_handler(result=ModelResponse(), standard_logging_object=payload) @@ -3731,7 +3756,9 @@ def test_sentry_event_scrubber_initialization(monkeypatch): call_args = mock_init.call_args[1] assert call_args["send_default_pii"] is False assert call_args["event_scrubber"].recursive is True - assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {name.lower() for name in call_args["event_scrubber"].denylist} + assert {name.lower() for name in SENTRY_PII_DENYLIST} <= { + name.lower() for name in call_args["event_scrubber"].denylist + } assert call_args["before_send"] is call_args["before_send_transaction"] @@ -3748,7 +3775,9 @@ def test_sentry_send_default_pii_opt_in(monkeypatch): call_args = mock_init.call_args[1] assert call_args["send_default_pii"] is True - assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {name.lower() for name in call_args["event_scrubber"].denylist} + assert not {name.lower() for name in SENTRY_PII_DENYLIST} & { + name.lower() for name in call_args["event_scrubber"].denylist + } def test_get_masked_values(): @@ -5018,12 +5047,10 @@ def test_reported_cost_respects_configured_pricing_and_additional_charges( else {} ), } - } - } - ) - logging_obj.model_call_details.update( - {"additional_response_cost": additional_cost, "cache_hit": cache_hit} + }, + }, ) + logging_obj.model_call_details.update({"additional_response_cost": additional_cost, "cache_hit": cache_hit}) logging_obj._provider_reported_cost = reported_cost response: Final = ModelResponse( model=pricing_model, @@ -6512,6 +6539,31 @@ def test_handle_anthropic_messages_response_logging_reparses_the_served_body_wit assert logged.usage.completion_tokens == 7 # type: ignore[attr-defined] +def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction_response(): + logging_obj = _anthropic_messages_logging_obj() + logging_obj.optional_params = {} + compaction_block = {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"} + + logged = logging_obj._handle_anthropic_messages_response_logging( + result={ + "id": "msg_compacted", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [compaction_block], + "stop_reason": "compaction", + "stop_sequence": None, + "usage": {"input_tokens": 410, "output_tokens": 95}, + } + ) + + assert isinstance(logged, ModelResponse) + assert logged.id == "msg_compacted" + assert logged.usage.prompt_tokens == 410 # type: ignore[attr-defined] + assert logged.usage.completion_tokens == 95 # type: ignore[attr-defined] + assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block] # type: ignore[union-attr] + + def test_handle_anthropic_messages_response_logging_passes_model_response_through(): """Anthropic-native path already yields a ModelResponse; it must be returned unchanged.""" logging_obj = _anthropic_messages_logging_obj() @@ -6950,8 +7002,11 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe class SyncRecorder(CustomLogger): def log_success_event( - self, kwargs: dict[str, object], response_obj: object, - start_time: dt.datetime, end_time: dt.datetime, + self, + kwargs: dict[str, object], + response_obj: object, + start_time: dt.datetime, + end_time: dt.datetime, ) -> None: events.put((response_obj, kwargs.get("response_cost"))) logged.set() @@ -6984,11 +7039,11 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe assert logging_obj.model_call_details["response_cost"] > 0 assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 - assert events.qsize() == 2 assert events.get_nowait() == (in_progress, None) assert events.get_nowait() == (completed, logging_obj.model_call_details["response_cost"]) + @pytest.mark.asyncio async def test_background_interaction_completion_prices_the_settled_body_itself(): """ @@ -8022,7 +8077,9 @@ def test_debugging_log_with_json_logs_tolerates_missing_headers(logging_obj, mon logging_obj.litellm_request_debug = True with patch("litellm.litellm_core_utils.litellm_logging.verbose_logger.warning") as warning: - logging_obj._print_llm_call_debugging_log(api_base="https://api.openai.com/v1", headers=None, additional_args={}) + logging_obj._print_llm_call_debugging_log( + api_base="https://api.openai.com/v1", headers=None, additional_args={} + ) assert "https://api.openai.com/v1" in warning.call_args.kwargs["extra"]["api_base"] @@ -10159,8 +10216,6 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() assert logging_obj.response_cost_calculator(result=assembled) == 0.0042 - - def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): logging_obj: Final = _responses_stream_logging_obj() inner_response: Final = ResponsesAPIResponse( @@ -10390,9 +10445,7 @@ def test_standard_logging_retries(): ] ) - with patch.object( - customHandler, "log_failure_event", new=MagicMock() - ) as mock_client: + with patch.object(customHandler, "log_failure_event", new=MagicMock()) as mock_client: try: router.completion( model="gpt-3.5-turbo", @@ -10404,19 +10457,10 @@ def test_standard_logging_retries(): pass assert mock_client.call_count == 2 + assert mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"] is not None assert ( - mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][ - "trace_id" - ] - is not None - ) - assert ( - mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][ - "trace_id" - ] - == mock_client.call_args_list[1].kwargs["kwargs"][ - "standard_logging_object" - ]["trace_id"] + mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"] + == mock_client.call_args_list[1].kwargs["kwargs"]["standard_logging_object"]["trace_id"] ) @@ -10683,9 +10727,7 @@ def test_signoz_dispatch_requires_an_endpoint(monkeypatch): custom_logger_init_args={}, ) assert created is None - assert not [ - cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz" - ] + assert not [cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz"] finally: logging_module._in_memory_loggers.clear() monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) @@ -10827,7 +10869,11 @@ def test_standard_logging_payload_reports_the_deployment_entry_the_request_was_b } } logging_obj.update_environment_variables( - model="gpt-5.4-nano", user="", optional_params={}, litellm_params=litellm_params, custom_llm_provider="openai" + model="gpt-5.4-nano", + user="", + optional_params={}, + litellm_params=litellm_params, + custom_llm_provider="openai", ) response: Final = ModelResponse( model="gpt-5.4-nano", @@ -11139,10 +11185,7 @@ def test_enterprise_alerting_loggers_falls_back_without_disabling_callback_contr ) logging_module._enterprise_alerting_loggers.cache_clear() for module_name in tuple(sys.modules): - if any( - module_name == prefix or module_name.startswith(f"{prefix}.") - for prefix in module_prefixes - ): + if any(module_name == prefix or module_name.startswith(f"{prefix}.") for prefix in module_prefixes): monkeypatch.delitem(sys.modules, module_name, raising=False) monkeypatch.setattr(sys, "meta_path", [BlockPagerDutyFinder(), *sys.meta_path]) @@ -11262,13 +11305,16 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest_asyncio.fixture(loop_scope="function") async def drain_logging_worker(isolate_litellm_state: None) -> AsyncIterator[None]: yield await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS) + LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS: Final = LOGGING_WORKER_MAX_TIME_PER_COROUTINE + 5.0 + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -11306,6 +11352,7 @@ def isolate_litellm_state(): if attr in _DEFAULTS: setattr(litellm, attr, _DEFAULTS[attr]) + _LIST_ATTRS = ( "callbacks", "success_callback", @@ -11333,6 +11380,7 @@ _SCALAR_ATTRS = ( _DEFAULTS: dict = {} + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -11355,6 +11403,7 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "response_obj,expected_values", @@ -11402,6 +11451,7 @@ def test_get_usage(response_obj, expected_values): assert usage.completion_tokens == expected_values[1] assert usage.total_tokens == expected_values[2] + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_usage_from_image_generation_response(): """ @@ -11449,6 +11499,7 @@ def test_get_usage_from_image_generation_response(): assert usage.completion_tokens_details.image_tokens == 272 assert usage.completion_tokens_details.text_tokens == 100 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_additional_headers(): additional_headers = { @@ -11489,10 +11540,12 @@ def test_get_additional_headers(): assert additional_logging_headers.get("llm_provider-request-id") == "req_01F6CycZZPSHKRCCctcS1Vto" assert additional_logging_headers.get("llm_provider-anthropic-ratelimit-requests-reset") == "2024-10-29T23:57:40Z" + def all_fields_present(standard_logging_metadata: StandardLoggingMetadata): for field in StandardLoggingMetadata.__annotations__.keys(): assert field in standard_logging_metadata + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "metadata_key, metadata_value", @@ -11524,6 +11577,7 @@ def test_get_standard_logging_metadata(metadata_key, metadata_value): # Assert that the specific metadata field is set correctly assert standard_logging_metadata[metadata_key] == metadata_value + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_user_api_key_hash(): valid_hash = "a" * 64 # 64 character string @@ -11531,6 +11585,7 @@ def test_get_standard_logging_metadata_user_api_key_hash(): result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata) assert result["user_api_key_hash"] == valid_hash + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_invalid_user_api_key(): invalid_hash = "not_a_valid_hash" @@ -11539,6 +11594,7 @@ def test_get_standard_logging_metadata_invalid_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_non_string_user_api_key(): """Non-string user_api_key should not be set as user_api_key_hash.""" @@ -11547,6 +11603,7 @@ def test_get_standard_logging_metadata_non_string_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_none_user_api_key(): """None user_api_key should not be set as user_api_key_hash.""" @@ -11555,6 +11612,7 @@ def test_get_standard_logging_metadata_none_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_invalid_keys(): metadata = { @@ -11568,6 +11626,7 @@ def test_get_standard_logging_metadata_invalid_keys(): assert "invalid_key" not in result assert "another_invalid_key" not in result + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cleanup_timestamps(): """Test cleanup_timestamps with different input types""" @@ -11602,6 +11661,7 @@ def test_cleanup_timestamps(): with pytest.raises(ValueError, match="start_time is required, got=invalid of type "): StandardLoggingPayloadSetup.cleanup_timestamps("invalid", end_float, completion_float) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_model_cost_information(): """Test get_model_cost_information with different inputs""" @@ -11634,6 +11694,7 @@ def test_get_model_cost_information(): # assert all fields in StandardLoggingModelInformation are present assert all(field in result for field in StandardLoggingModelInformation.__annotations__) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_model_cost_information_custom_pricing_uses_base_model(): result = StandardLoggingPayloadSetup.get_model_cost_information( @@ -11645,6 +11706,7 @@ def test_get_model_cost_information_custom_pricing_uses_base_model(): assert result["model_map_value"] is not None assert result["model_map_key"] != "invoke_test_claude" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_deployment_when_no_base_model(): """metadata["deployment"] is used for cost-map lookup when base_model is not set.""" @@ -11702,6 +11764,7 @@ def test_standard_logging_payload_uses_deployment_when_no_base_model(): assert payload["model_map_information"]["model_map_value"] is not None assert payload["model_map_information"]["model_map_key"] != "invoke_test_claude" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_hidden_params(): """Test get_hidden_params with different inputs""" @@ -11737,6 +11800,7 @@ def test_get_hidden_params(): # assert all fields in StandardLoggingHiddenParams are present assert all(field in result for field in StandardLoggingHiddenParams.__annotations__) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_final_response_obj(): """Test get_final_response_obj with different input types and redaction scenarios""" @@ -11768,6 +11832,7 @@ def test_get_final_response_obj(): # Reset litellm.turn_off_message_logging to its original value litellm.turn_off_message_logging = False + def testget_standard_logging_payload_trace_id(): """Test get_standard_logging_payload_trace_id with different input scenarios""" # Test case 1: When litellm_trace_id is provided in litellm_params @@ -11805,6 +11870,7 @@ def testget_standard_logging_payload_trace_id(): assert result == "12345" assert isinstance(result, str) + def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on(monkeypatch): """With request_correlation_in_logs on, an explicit litellm_trace_id wins over litellm_session_id.""" from unittest.mock import MagicMock @@ -11819,6 +11885,7 @@ def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on( ) assert result == "the-trace-id" + def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_off(monkeypatch): """With request_correlation_in_logs off (default), legacy behavior is preserved: litellm_session_id still wins over litellm_trace_id.""" @@ -11834,6 +11901,7 @@ def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_o ) assert result == "the-session-id" + def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch): """Test get_standard_logging_payload_session_id with different input scenarios, flag enabled""" from unittest.mock import MagicMock @@ -11886,6 +11954,7 @@ def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch): ) assert result == "" + def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch): """When request_correlation_in_logs is off (default), session_id is always empty, even if litellm_session_id was explicitly supplied - preserves the pre-existing @@ -11902,6 +11971,7 @@ def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch) ) assert result == "" + @pytest.fixture def restore_verbose_logger_level() -> Iterator[None]: original_level: Final = verbose_logger.level @@ -11944,6 +12014,7 @@ def test_truncate_standard_logging_payload(): assert len(str(truncated["response"])) < 10_500 assert len(str(truncated["error_str"])) < 10_500 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact(): """A payload built with only some of its fields comes back with exactly those keys and values""" @@ -11952,12 +12023,14 @@ def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact(): assert _custom_logger.truncate_standard_logging_payload_content(partial_payload) == partial_payload + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_strip_trailing_slash(): common_api_base = "https://api.test.com" assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base + "/") == common_api_base assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base) == common_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_error_information(): """Test get_error_information with different types of exceptions""" @@ -11994,6 +12067,7 @@ def test_get_error_information(): assert result["llm_provider"] == "openai" assert result["error_message"] == "litellm.RateLimitError: Test error" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_response_time(): """Test get_response_time with different streaming scenarios""" @@ -12029,6 +12103,7 @@ def test_get_response_time(): # For streaming, should return completion_start_time - start_time assert response_time == 2.0 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "metadata, expected_requester_metadata", @@ -12051,6 +12126,7 @@ def test_standard_logging_metadata_requester_metadata(metadata, expected_request result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata) assert result["requester_metadata"] == expected_requester_metadata + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cost_breakdown_in_standard_logging_payload(): """ @@ -12140,6 +12216,7 @@ def test_cost_breakdown_in_standard_logging_payload(): print("✅ Cost breakdown test passed!") + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cost_breakdown_missing_in_standard_logging_payload(): """ @@ -12200,6 +12277,7 @@ def test_cost_breakdown_missing_in_standard_logging_payload(): print("✅ Cost breakdown missing test passed!") + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "use_combined_usage_object", @@ -12287,6 +12365,7 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object): assert usage_obj["completion_tokens"] == 58 assert usage_obj["total_tokens"] == 100 + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_actual_model_for_azure_router(): from litellm.litellm_core_utils.litellm_logging import ( @@ -12335,6 +12414,7 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router(): assert payload is not None assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_actual_model_for_azure_router_with_underscore(): from litellm.litellm_core_utils.litellm_logging import ( @@ -12383,6 +12463,7 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router_with_unders assert payload is not None assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_basic(): """ @@ -12414,6 +12495,7 @@ def test_merge_litellm_metadata_basic(): assert result["model_info"] == {"id": "model-123"} assert result["tags"] == ["tag1", "tag2"] + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_precedence(): """ @@ -12440,6 +12522,7 @@ def test_merge_litellm_metadata_precedence(): # litellm_metadata values should only be included if not in metadata assert result["model_group"] == "gpt-4-group" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_skip_non_serializable(): """ @@ -12472,6 +12555,7 @@ def test_merge_litellm_metadata_skip_non_serializable(): assert result["safe_field"] == "safe_value" assert result["model_group"] == "gpt-4-group" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_empty_params(): """ @@ -12507,6 +12591,7 @@ def test_merge_litellm_metadata_empty_params(): result = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) assert result == {} + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_bedrock_passthrough_scenario(): """ @@ -12566,8 +12651,10 @@ def test_merge_litellm_metadata_bedrock_passthrough_scenario(): # Verify total number of fields (9 user fields + 4 model fields = 13) assert len(result) == 13 + service_logger = ServiceLogging() + def setup_logging(): return Logging( model="gpt-5.5", @@ -12657,6 +12744,7 @@ def test_get_callback_name(): # Test string callback assert logging._get_callback_name("callback_string") == "callback_string" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_is_internal_litellm_proxy_callback(): """ @@ -12677,6 +12765,7 @@ def test_is_internal_litellm_proxy_callback(): # Test string callback assert logging._is_internal_litellm_proxy_callback("callback_string") == False + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_should_run_sync_callbacks_for_async_calls(): """ @@ -12702,6 +12791,7 @@ def test_should_run_sync_callbacks_for_async_calls(): litellm.success_callback = [PROXY_MaxIterationsHandler] assert logging._should_run_sync_callbacks_for_async_calls() == False + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_remove_internal_litellm_callbacks(): logging = setup_logging() @@ -12723,6 +12813,7 @@ def test_remove_internal_litellm_callbacks(): assert PROXY_MaxIterationsHandler not in filtered assert PROXY_CacheControlCheck not in filtered + @pytest.mark.asyncio async def test_background_interaction_completion_logs_while_in_progress_handler_is_parked(monkeypatch): """ @@ -12825,14 +12916,9 @@ class CompletionCustomHandler( assert metadata_value is None or isinstance(metadata_value, dict) if metadata_value is not None: if litellm.turn_off_message_logging is True: - assert ( - metadata_value["raw_request"] - is LiteLLMCommonStrings.redacted_by_litellm.value - ) + assert metadata_value["raw_request"] is LiteLLMCommonStrings.redacted_by_litellm.value else: - assert "raw_request" not in metadata_value or isinstance( - metadata_value["raw_request"], str - ) + assert "raw_request" not in metadata_value or isinstance(metadata_value["raw_request"], str) except Exception: print(f"Assertion Error: {traceback.format_exc()}") self.errors.append(traceback.format_exc()) @@ -12881,23 +12967,18 @@ class CompletionCustomHandler( assert isinstance(response_obj, litellm.ModelResponseStream) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["start_time"], (datetime, type(None))) assert isinstance(kwargs["stream"], bool) assert isinstance(kwargs["user"], (str, type(None))) - assert ( - isinstance(kwargs["input"], list) - and isinstance(kwargs["input"][0], dict) - ) or isinstance(kwargs["input"], (dict, str)) + assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance( + kwargs["input"], (dict, str) + ) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) ) @@ -12930,9 +13011,7 @@ class CompletionCustomHandler( ) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["litellm_params"]["api_base"], str) @@ -12942,10 +13021,7 @@ class CompletionCustomHandler( assert isinstance(kwargs["user"], (str, type(None))) assert ( isinstance(kwargs["input"], list) - and ( - isinstance(kwargs["input"][0], dict) - or isinstance(kwargs["input"][0], str) - ) + and (isinstance(kwargs["input"][0], dict) or isinstance(kwargs["input"][0], str)) ) or isinstance(kwargs["input"], (dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert isinstance( @@ -12973,9 +13049,7 @@ class CompletionCustomHandler( assert response_obj == None ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) @@ -12983,15 +13057,12 @@ class CompletionCustomHandler( assert isinstance(kwargs["start_time"], (datetime, type(None))) assert isinstance(kwargs["stream"], bool) assert isinstance(kwargs["user"], (str, type(None))) - assert ( - isinstance(kwargs["input"], list) - and isinstance(kwargs["input"][0], dict) - ) or isinstance(kwargs["input"], (dict, str)) + assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance( + kwargs["input"], (dict, str) + ) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or kwargs["original_response"] == None ) assert isinstance(kwargs["additional_args"], (dict, type(None))) @@ -13009,9 +13080,7 @@ class CompletionCustomHandler( assert isinstance(messages, list) and isinstance(messages[0], dict) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance( - kwargs["messages"][0], dict - ) + assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["start_time"], (datetime, type(None))) @@ -13023,9 +13092,7 @@ class CompletionCustomHandler( async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: - print( - "in async_log_success_event", kwargs, response_obj, start_time, end_time - ) + print("in async_log_success_event", kwargs, response_obj, start_time, end_time) self.states.append("async_success") ## START TIME assert isinstance(start_time, datetime) @@ -13054,9 +13121,7 @@ class CompletionCustomHandler( assert isinstance(kwargs["input"], (list, dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) ) @@ -13088,9 +13153,7 @@ class CompletionCustomHandler( assert isinstance(kwargs["input"], (list, str, dict)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance( - kwargs["original_response"], (str, litellm.CustomStreamWrapper) - ) + isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) or kwargs["original_response"] == None From 90b5ccb89ce21c2e16c8ad438ed3b5d2cce5ae0e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:14:15 -0700 Subject: [PATCH 09/12] fix(vertex_ai,azure_ai): forward compact-2026-09-04 to Azure AI Foundry, merge Vertex body betas, and redact compaction summaries Azure AI Foundry needs the compact-2026-09-04 header for compaction, a replayed compaction block, and count tokens, so the beta map, the native compaction providers, and the Rust beta table now allow it there. Vertex keeps a client's body betas next to the ones litellm adds, as the Anthropic route does, and turn_off_message_logging now redacts compaction summaries too --- .../src/providers/anthropic/beta.rs | 1 - .../src/azure_ai/messages/transformation.rs | 5 + litellm/anthropic_beta_headers_config.json | 1 + litellm/litellm_core_utils/redact_messages.py | 6 +- litellm/llms/anthropic/common_utils.py | 2 +- .../pass_through/messages/transformation.py | 3 +- .../anthropic/count_tokens/handler.py | 1 + .../anthropic/count_tokens/transformation.py | 11 +- .../anthropic/transformation.py | 2 +- .../test_litellm_logging.py | 234 ++++++++---------- .../test_redact_messages.py | 1 + .../anthropic/test_anthropic_common_utils.py | 4 +- .../count_tokens/test_token_counter.py | 33 +++ ...azure_anthropic_messages_transformation.py | 46 ++++ .../test_azure_anthropic_transformation.py | 30 +++ ...partner_models_anthropic_transformation.py | 20 +- .../test_anthropic_beta_headers_filtering.py | 2 +- 17 files changed, 251 insertions(+), 151 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 779c5f9d00e..e731a749026 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -208,7 +208,6 @@ impl AnthropicBeta { BetaProvider::Databricks, ], Self::Compact20260904 => &[ - BetaProvider::AzureAi, BetaProvider::BedrockConverse, BetaProvider::Bedrock, BetaProvider::BedrockMantle, diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index bf934fb4d4c..d578acea68c 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -545,6 +545,11 @@ mod tests { &[("anthropic-beta", "web-search-2025-03-05")], &[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")] )] + #[case::compaction_needs_the_compact_2026_09_04_beta( + json!({"compaction": {"type": "summarize"}}), + &[], + &[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")] + )] #[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])] #[case::safeguards_need_the_dangerous_tool_use_beta( json!({"safeguards": [{"type": "dangerous_tool_use"}]}), diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 340f7cf9fa4..8cd6937510b 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -46,6 +46,7 @@ "bash_20250124": null, "code-execution-2025-08-25": "code-execution-2025-08-25", "compact-2026-01-12": null, + "compact-2026-09-04": "compact-2026-09-04", "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index c2cca5739cf..ec42c0e059a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -147,7 +147,11 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[ return { **fields, **({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}), - **{key: None for key in ("thinking_blocks", "reasoningContentBlocks", "citations") if key in fields}, + **{ + key: None + for key in ("thinking_blocks", "reasoningContentBlocks", "citations", "compaction_blocks") + if key in fields + }, } diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 51ec1312f95..26e5dab890e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -119,7 +119,7 @@ _DECODED_JSON: Final = TypeAdapter(object) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") -NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "vertex_ai"}) +NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"}) def is_anthropic_messages_url(url: str) -> bool: diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index a71cd72b768..a76f857270c 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -28,6 +28,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( + NATIVE_COMPACTION_PROVIDERS, AnthropicError, AnthropicModelInfo, merge_anthropic_beta_headers, @@ -99,7 +100,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "tool_choice", "thinking", "context_management", - *(("compaction",) if self._resolved_provider == "anthropic" else ()), + *(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()), "output_format", "inference_geo", "speed", diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 53734a77f3c..d0387218168 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -79,6 +79,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): headers: Final = self.get_required_headers( api_key=api_key, litellm_params=litellm_params, + messages=messages, ) if not {"x-api-key", "Authorization"} & headers.keys(): raise AnthropicError( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py index 5946a1f4480..0e42de201d8 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py @@ -4,13 +4,16 @@ Azure AI Anthropic CountTokens API transformation logic. Extends the base Anthropic CountTokens transformation with Azure authentication. """ +from collections.abc import Sequence from typing import Final from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers, requires_native_compaction_beta from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.router import GenericLiteLLMParams @@ -26,6 +29,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): self, api_key: str | None, litellm_params: dict[str, object] | None = None, + messages: Sequence[object] = (), ) -> dict[str, str]: """ Get the required headers for the Azure AI Anthropic CountTokens API. @@ -44,7 +48,12 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): headers: Final = { "Content-Type": "application/json", "anthropic-version": "2023-06-01", - "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, + "anthropic-beta": merge_anthropic_beta_headers( + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value + if requires_native_compaction_beta("azure_ai", {}, messages) + else None, + ), **({"x-api-key": api_key} if api_key else {}), } diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 18022f8d51d..bfdca373803 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -183,7 +183,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): _NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params ) - body_betas: Final = (injected_betas or client_body_betas) - _VERTEX_HEADER_ONLY_BETAS + body_betas: Final = (injected_betas | client_body_betas) - _VERTEX_HEADER_ONLY_BETAS if body_betas: data["anthropic_beta"] = sorted(body_betas) elif "anthropic_beta" in data: diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 608b95bbaeb..96d9edb90d9 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -75,7 +75,6 @@ import inspect from typing import List, Optional from litellm.types.utils import LiteLLMCommonStrings - @pytest.fixture def logging_obj(): return LitellmLogging( @@ -346,10 +345,8 @@ async def test_mcp_native_structured_replacement_must_match_returned_content( ) logging_obj.dynamic_success_callbacks = [NativeReplacement()] returned = await logging_obj.async_post_mcp_tool_call_hook( - kwargs={"original_response": result}, - response_obj=result, - start_time=datetime.datetime.now(), - end_time=datetime.datetime.now(), + kwargs={"original_response": result}, response_obj=result, + start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), ) assert returned is result assert result.content == [TextContent(type="text", text="native-safe" if same_content else "final-safe")] @@ -358,6 +355,7 @@ async def test_mcp_native_structured_replacement_must_match_returned_content( assert "SECRET-1234" not in result.model_dump_json() + @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["none", "wrapper", "exception"]) @pytest.mark.parametrize("structured", [False, True]) @@ -375,10 +373,8 @@ async def test_mcp_direct_content_edit_invalidates_stale_structured_data(logging ) logging_obj.dynamic_success_callbacks = [DirectRedactor()] returned = await logging_obj.async_post_mcp_tool_call_hook( - kwargs={"original_response": result}, - response_obj=result, - start_time=datetime.datetime.now(), - end_time=datetime.datetime.now(), + kwargs={"original_response": result}, response_obj=result, + start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), ) assert returned is result assert result.content == [TextContent(type="text", text="[REDACTED]")] @@ -2543,6 +2539,7 @@ async def test_completed_success_selects_one_implemented_hook( assert (thread_id == loop_thread) is (kind != "sync") + @pytest.mark.parametrize("for_async_request", [False, True]) @pytest.mark.parametrize("private_first", [False, True]) @pytest.mark.parametrize("response_kind", ["responses", "completion"]) @@ -3230,13 +3227,8 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g class RecordingShadowLogger(ShadowEvalLogger): async def async_log_success_event( - self, - kwargs: Mapping[str, object], - response_obj: object, - start_time: object, - end_time: object, - *, - guardrail_snapshot: GuardrailRequestSnapshot | None = None, + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, + end_time: object, *, guardrail_snapshot: GuardrailRequestSnapshot | None = None, ) -> None: shadow_snapshots.append(guardrail_snapshot) await super().async_log_success_event( @@ -3245,20 +3237,13 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g class RecordingLogger(CustomLogger): async def async_log_success_event( - self, - kwargs: Mapping[str, object], - response_obj: object, - start_time: object, - end_time: object, + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object, ) -> None: other_payloads.append(kwargs) class LoggingGuardrail(CustomGuardrail): async def async_logging_hook( - self, - kwargs: dict[str, object], - result: object, - call_type: str, + self, kwargs: dict[str, object], result: object, call_type: str, ) -> tuple[dict[str, object], object]: hook_snapshots.append(logging_obj.shadow_eval_request_snapshot) if hook_mode == "raises": @@ -3270,36 +3255,26 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g "user_api_key_hash": "test-key", } snapshot: Final = GuardrailRequestSnapshot.capture( - {"messages": [{"role": "user", "content": "snapshot-only"}]}, - metadata, + {"messages": [{"role": "user", "content": "snapshot-only"}]}, metadata, ) assert snapshot is not None shadow: Final = RecordingShadowLogger(prisma_provider=no_prisma, jobs_cache=InMemoryCache()) guardrail: Final = LoggingGuardrail( - guardrail_name="late-mask", - default_on=True, + guardrail_name="late-mask", default_on=True, event_hook=GuardrailEventHooks.pre_call if hook_mode == "disabled" else GuardrailEventHooks.logging_only, ) monkeypatch.setattr(litellm, "_async_success_callback", []) logging_obj: Final = LitellmLogging( - model="test-model", - messages=[], - stream=stream, - call_type="anthropic_messages", - start_time=datetime.datetime.now(), - litellm_call_id="private-snapshot", - function_id="private-snapshot", + model="test-model", messages=[], stream=stream, call_type="anthropic_messages", + start_time=datetime.datetime.now(), litellm_call_id="private-snapshot", function_id="private-snapshot", dynamic_async_success_callbacks=[shadow, RecordingLogger(), guardrail], ) logging_obj.update_messages([{"role": "user", "content": "logged input"}]) logging_obj.update_environment_variables(litellm_params={"metadata": metadata}, optional_params={}) logging_obj.shadow_eval_request_snapshot = snapshot payload: Final = { - "id": "private-snapshot", - "call_type": "anthropic_messages", - "metadata": metadata, - "model_group": "test-model", - "model_parameters": {}, + "id": "private-snapshot", "call_type": "anthropic_messages", "metadata": metadata, + "model_group": "test-model", "model_parameters": {}, } await logging_obj.async_success_handler(result=ModelResponse(), standard_logging_object=payload) @@ -3756,9 +3731,7 @@ def test_sentry_event_scrubber_initialization(monkeypatch): call_args = mock_init.call_args[1] assert call_args["send_default_pii"] is False assert call_args["event_scrubber"].recursive is True - assert {name.lower() for name in SENTRY_PII_DENYLIST} <= { - name.lower() for name in call_args["event_scrubber"].denylist - } + assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {name.lower() for name in call_args["event_scrubber"].denylist} assert call_args["before_send"] is call_args["before_send_transaction"] @@ -3775,9 +3748,7 @@ def test_sentry_send_default_pii_opt_in(monkeypatch): call_args = mock_init.call_args[1] assert call_args["send_default_pii"] is True - assert not {name.lower() for name in SENTRY_PII_DENYLIST} & { - name.lower() for name in call_args["event_scrubber"].denylist - } + assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {name.lower() for name in call_args["event_scrubber"].denylist} def test_get_masked_values(): @@ -5047,10 +5018,12 @@ def test_reported_cost_respects_configured_pricing_and_additional_charges( else {} ), } - }, - }, + } + } + ) + logging_obj.model_call_details.update( + {"additional_response_cost": additional_cost, "cache_hit": cache_hit} ) - logging_obj.model_call_details.update({"additional_response_cost": additional_cost, "cache_hit": cache_hit}) logging_obj._provider_reported_cost = reported_cost response: Final = ModelResponse( model=pricing_model, @@ -7002,11 +6975,8 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe class SyncRecorder(CustomLogger): def log_success_event( - self, - kwargs: dict[str, object], - response_obj: object, - start_time: dt.datetime, - end_time: dt.datetime, + self, kwargs: dict[str, object], response_obj: object, + start_time: dt.datetime, end_time: dt.datetime, ) -> None: events.put((response_obj, kwargs.get("response_cost"))) logged.set() @@ -7039,11 +7009,11 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe assert logging_obj.model_call_details["response_cost"] > 0 assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + assert events.qsize() == 2 assert events.get_nowait() == (in_progress, None) assert events.get_nowait() == (completed, logging_obj.model_call_details["response_cost"]) - @pytest.mark.asyncio async def test_background_interaction_completion_prices_the_settled_body_itself(): """ @@ -8077,9 +8047,7 @@ def test_debugging_log_with_json_logs_tolerates_missing_headers(logging_obj, mon logging_obj.litellm_request_debug = True with patch("litellm.litellm_core_utils.litellm_logging.verbose_logger.warning") as warning: - logging_obj._print_llm_call_debugging_log( - api_base="https://api.openai.com/v1", headers=None, additional_args={} - ) + logging_obj._print_llm_call_debugging_log(api_base="https://api.openai.com/v1", headers=None, additional_args={}) assert "https://api.openai.com/v1" in warning.call_args.kwargs["extra"]["api_base"] @@ -10216,6 +10184,8 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() assert logging_obj.response_cost_calculator(result=assembled) == 0.0042 + + def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): logging_obj: Final = _responses_stream_logging_obj() inner_response: Final = ResponsesAPIResponse( @@ -10445,7 +10415,9 @@ def test_standard_logging_retries(): ] ) - with patch.object(customHandler, "log_failure_event", new=MagicMock()) as mock_client: + with patch.object( + customHandler, "log_failure_event", new=MagicMock() + ) as mock_client: try: router.completion( model="gpt-3.5-turbo", @@ -10457,10 +10429,19 @@ def test_standard_logging_retries(): pass assert mock_client.call_count == 2 - assert mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"] is not None assert ( - mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"] - == mock_client.call_args_list[1].kwargs["kwargs"]["standard_logging_object"]["trace_id"] + mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][ + "trace_id" + ] + is not None + ) + assert ( + mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][ + "trace_id" + ] + == mock_client.call_args_list[1].kwargs["kwargs"][ + "standard_logging_object" + ]["trace_id"] ) @@ -10727,7 +10708,9 @@ def test_signoz_dispatch_requires_an_endpoint(monkeypatch): custom_logger_init_args={}, ) assert created is None - assert not [cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz"] + assert not [ + cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz" + ] finally: logging_module._in_memory_loggers.clear() monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) @@ -10869,11 +10852,7 @@ def test_standard_logging_payload_reports_the_deployment_entry_the_request_was_b } } logging_obj.update_environment_variables( - model="gpt-5.4-nano", - user="", - optional_params={}, - litellm_params=litellm_params, - custom_llm_provider="openai", + model="gpt-5.4-nano", user="", optional_params={}, litellm_params=litellm_params, custom_llm_provider="openai" ) response: Final = ModelResponse( model="gpt-5.4-nano", @@ -11185,7 +11164,10 @@ def test_enterprise_alerting_loggers_falls_back_without_disabling_callback_contr ) logging_module._enterprise_alerting_loggers.cache_clear() for module_name in tuple(sys.modules): - if any(module_name == prefix or module_name.startswith(f"{prefix}.") for prefix in module_prefixes): + if any( + module_name == prefix or module_name.startswith(f"{prefix}.") + for prefix in module_prefixes + ): monkeypatch.delitem(sys.modules, module_name, raising=False) monkeypatch.setattr(sys, "meta_path", [BlockPagerDutyFinder(), *sys.meta_path]) @@ -11305,16 +11287,13 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) - @pytest_asyncio.fixture(loop_scope="function") async def drain_logging_worker(isolate_litellm_state: None) -> AsyncIterator[None]: yield await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS) - LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS: Final = LOGGING_WORKER_MAX_TIME_PER_COROUTINE + 5.0 - @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -11352,7 +11331,6 @@ def isolate_litellm_state(): if attr in _DEFAULTS: setattr(litellm, attr, _DEFAULTS[attr]) - _LIST_ATTRS = ( "callbacks", "success_callback", @@ -11380,7 +11358,6 @@ _SCALAR_ATTRS = ( _DEFAULTS: dict = {} - @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -11403,7 +11380,6 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "response_obj,expected_values", @@ -11451,7 +11427,6 @@ def test_get_usage(response_obj, expected_values): assert usage.completion_tokens == expected_values[1] assert usage.total_tokens == expected_values[2] - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_usage_from_image_generation_response(): """ @@ -11499,7 +11474,6 @@ def test_get_usage_from_image_generation_response(): assert usage.completion_tokens_details.image_tokens == 272 assert usage.completion_tokens_details.text_tokens == 100 - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_additional_headers(): additional_headers = { @@ -11540,12 +11514,10 @@ def test_get_additional_headers(): assert additional_logging_headers.get("llm_provider-request-id") == "req_01F6CycZZPSHKRCCctcS1Vto" assert additional_logging_headers.get("llm_provider-anthropic-ratelimit-requests-reset") == "2024-10-29T23:57:40Z" - def all_fields_present(standard_logging_metadata: StandardLoggingMetadata): for field in StandardLoggingMetadata.__annotations__.keys(): assert field in standard_logging_metadata - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "metadata_key, metadata_value", @@ -11577,7 +11549,6 @@ def test_get_standard_logging_metadata(metadata_key, metadata_value): # Assert that the specific metadata field is set correctly assert standard_logging_metadata[metadata_key] == metadata_value - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_user_api_key_hash(): valid_hash = "a" * 64 # 64 character string @@ -11585,7 +11556,6 @@ def test_get_standard_logging_metadata_user_api_key_hash(): result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata) assert result["user_api_key_hash"] == valid_hash - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_invalid_user_api_key(): invalid_hash = "not_a_valid_hash" @@ -11594,7 +11564,6 @@ def test_get_standard_logging_metadata_invalid_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_non_string_user_api_key(): """Non-string user_api_key should not be set as user_api_key_hash.""" @@ -11603,7 +11572,6 @@ def test_get_standard_logging_metadata_non_string_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_none_user_api_key(): """None user_api_key should not be set as user_api_key_hash.""" @@ -11612,7 +11580,6 @@ def test_get_standard_logging_metadata_none_user_api_key(): all_fields_present(result) assert result["user_api_key_hash"] is None - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_invalid_keys(): metadata = { @@ -11626,7 +11593,6 @@ def test_get_standard_logging_metadata_invalid_keys(): assert "invalid_key" not in result assert "another_invalid_key" not in result - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cleanup_timestamps(): """Test cleanup_timestamps with different input types""" @@ -11661,7 +11627,6 @@ def test_cleanup_timestamps(): with pytest.raises(ValueError, match="start_time is required, got=invalid of type "): StandardLoggingPayloadSetup.cleanup_timestamps("invalid", end_float, completion_float) - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_model_cost_information(): """Test get_model_cost_information with different inputs""" @@ -11694,7 +11659,6 @@ def test_get_model_cost_information(): # assert all fields in StandardLoggingModelInformation are present assert all(field in result for field in StandardLoggingModelInformation.__annotations__) - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_model_cost_information_custom_pricing_uses_base_model(): result = StandardLoggingPayloadSetup.get_model_cost_information( @@ -11706,7 +11670,6 @@ def test_get_model_cost_information_custom_pricing_uses_base_model(): assert result["model_map_value"] is not None assert result["model_map_key"] != "invoke_test_claude" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_deployment_when_no_base_model(): """metadata["deployment"] is used for cost-map lookup when base_model is not set.""" @@ -11764,7 +11727,6 @@ def test_standard_logging_payload_uses_deployment_when_no_base_model(): assert payload["model_map_information"]["model_map_value"] is not None assert payload["model_map_information"]["model_map_key"] != "invoke_test_claude" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_hidden_params(): """Test get_hidden_params with different inputs""" @@ -11800,7 +11762,6 @@ def test_get_hidden_params(): # assert all fields in StandardLoggingHiddenParams are present assert all(field in result for field in StandardLoggingHiddenParams.__annotations__) - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_final_response_obj(): """Test get_final_response_obj with different input types and redaction scenarios""" @@ -11832,7 +11793,6 @@ def test_get_final_response_obj(): # Reset litellm.turn_off_message_logging to its original value litellm.turn_off_message_logging = False - def testget_standard_logging_payload_trace_id(): """Test get_standard_logging_payload_trace_id with different input scenarios""" # Test case 1: When litellm_trace_id is provided in litellm_params @@ -11870,7 +11830,6 @@ def testget_standard_logging_payload_trace_id(): assert result == "12345" assert isinstance(result, str) - def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on(monkeypatch): """With request_correlation_in_logs on, an explicit litellm_trace_id wins over litellm_session_id.""" from unittest.mock import MagicMock @@ -11885,7 +11844,6 @@ def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on( ) assert result == "the-trace-id" - def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_off(monkeypatch): """With request_correlation_in_logs off (default), legacy behavior is preserved: litellm_session_id still wins over litellm_trace_id.""" @@ -11901,7 +11859,6 @@ def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_o ) assert result == "the-session-id" - def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch): """Test get_standard_logging_payload_session_id with different input scenarios, flag enabled""" from unittest.mock import MagicMock @@ -11954,7 +11911,6 @@ def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch): ) assert result == "" - def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch): """When request_correlation_in_logs is off (default), session_id is always empty, even if litellm_session_id was explicitly supplied - preserves the pre-existing @@ -11971,7 +11927,6 @@ def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch) ) assert result == "" - @pytest.fixture def restore_verbose_logger_level() -> Iterator[None]: original_level: Final = verbose_logger.level @@ -12014,7 +11969,6 @@ def test_truncate_standard_logging_payload(): assert len(str(truncated["response"])) < 10_500 assert len(str(truncated["error_str"])) < 10_500 - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact(): """A payload built with only some of its fields comes back with exactly those keys and values""" @@ -12023,14 +11977,12 @@ def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact(): assert _custom_logger.truncate_standard_logging_payload_content(partial_payload) == partial_payload - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_strip_trailing_slash(): common_api_base = "https://api.test.com" assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base + "/") == common_api_base assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base) == common_api_base - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_error_information(): """Test get_error_information with different types of exceptions""" @@ -12067,7 +12019,6 @@ def test_get_error_information(): assert result["llm_provider"] == "openai" assert result["error_message"] == "litellm.RateLimitError: Test error" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_response_time(): """Test get_response_time with different streaming scenarios""" @@ -12103,7 +12054,6 @@ def test_get_response_time(): # For streaming, should return completion_start_time - start_time assert response_time == 2.0 - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "metadata, expected_requester_metadata", @@ -12126,7 +12076,6 @@ def test_standard_logging_metadata_requester_metadata(metadata, expected_request result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata) assert result["requester_metadata"] == expected_requester_metadata - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cost_breakdown_in_standard_logging_payload(): """ @@ -12216,7 +12165,6 @@ def test_cost_breakdown_in_standard_logging_payload(): print("✅ Cost breakdown test passed!") - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_cost_breakdown_missing_in_standard_logging_payload(): """ @@ -12277,7 +12225,6 @@ def test_cost_breakdown_missing_in_standard_logging_payload(): print("✅ Cost breakdown missing test passed!") - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "use_combined_usage_object", @@ -12365,7 +12312,6 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object): assert usage_obj["completion_tokens"] == 58 assert usage_obj["total_tokens"] == 100 - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_actual_model_for_azure_router(): from litellm.litellm_core_utils.litellm_logging import ( @@ -12414,7 +12360,6 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router(): assert payload is not None assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_actual_model_for_azure_router_with_underscore(): from litellm.litellm_core_utils.litellm_logging import ( @@ -12463,7 +12408,6 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router_with_unders assert payload is not None assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_basic(): """ @@ -12495,7 +12439,6 @@ def test_merge_litellm_metadata_basic(): assert result["model_info"] == {"id": "model-123"} assert result["tags"] == ["tag1", "tag2"] - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_precedence(): """ @@ -12522,7 +12465,6 @@ def test_merge_litellm_metadata_precedence(): # litellm_metadata values should only be included if not in metadata assert result["model_group"] == "gpt-4-group" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_skip_non_serializable(): """ @@ -12555,7 +12497,6 @@ def test_merge_litellm_metadata_skip_non_serializable(): assert result["safe_field"] == "safe_value" assert result["model_group"] == "gpt-4-group" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_empty_params(): """ @@ -12591,7 +12532,6 @@ def test_merge_litellm_metadata_empty_params(): result = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) assert result == {} - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_merge_litellm_metadata_bedrock_passthrough_scenario(): """ @@ -12651,10 +12591,8 @@ def test_merge_litellm_metadata_bedrock_passthrough_scenario(): # Verify total number of fields (9 user fields + 4 model fields = 13) assert len(result) == 13 - service_logger = ServiceLogging() - def setup_logging(): return Logging( model="gpt-5.5", @@ -12744,7 +12682,6 @@ def test_get_callback_name(): # Test string callback assert logging._get_callback_name("callback_string") == "callback_string" - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_is_internal_litellm_proxy_callback(): """ @@ -12765,7 +12702,6 @@ def test_is_internal_litellm_proxy_callback(): # Test string callback assert logging._is_internal_litellm_proxy_callback("callback_string") == False - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_should_run_sync_callbacks_for_async_calls(): """ @@ -12791,7 +12727,6 @@ def test_should_run_sync_callbacks_for_async_calls(): litellm.success_callback = [PROXY_MaxIterationsHandler] assert logging._should_run_sync_callbacks_for_async_calls() == False - @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_remove_internal_litellm_callbacks(): logging = setup_logging() @@ -12813,7 +12748,6 @@ def test_remove_internal_litellm_callbacks(): assert PROXY_MaxIterationsHandler not in filtered assert PROXY_CacheControlCheck not in filtered - @pytest.mark.asyncio async def test_background_interaction_completion_logs_while_in_progress_handler_is_parked(monkeypatch): """ @@ -12916,9 +12850,14 @@ class CompletionCustomHandler( assert metadata_value is None or isinstance(metadata_value, dict) if metadata_value is not None: if litellm.turn_off_message_logging is True: - assert metadata_value["raw_request"] is LiteLLMCommonStrings.redacted_by_litellm.value + assert ( + metadata_value["raw_request"] + is LiteLLMCommonStrings.redacted_by_litellm.value + ) else: - assert "raw_request" not in metadata_value or isinstance(metadata_value["raw_request"], str) + assert "raw_request" not in metadata_value or isinstance( + metadata_value["raw_request"], str + ) except Exception: print(f"Assertion Error: {traceback.format_exc()}") self.errors.append(traceback.format_exc()) @@ -12967,18 +12906,23 @@ class CompletionCustomHandler( assert isinstance(response_obj, litellm.ModelResponseStream) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) + assert isinstance(kwargs["messages"], list) and isinstance( + kwargs["messages"][0], dict + ) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["start_time"], (datetime, type(None))) assert isinstance(kwargs["stream"], bool) assert isinstance(kwargs["user"], (str, type(None))) - assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance( - kwargs["input"], (dict, str) - ) + assert ( + isinstance(kwargs["input"], list) + and isinstance(kwargs["input"][0], dict) + ) or isinstance(kwargs["input"], (dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) + isinstance( + kwargs["original_response"], (str, litellm.CustomStreamWrapper) + ) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) ) @@ -13011,7 +12955,9 @@ class CompletionCustomHandler( ) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) + assert isinstance(kwargs["messages"], list) and isinstance( + kwargs["messages"][0], dict + ) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["litellm_params"]["api_base"], str) @@ -13021,7 +12967,10 @@ class CompletionCustomHandler( assert isinstance(kwargs["user"], (str, type(None))) assert ( isinstance(kwargs["input"], list) - and (isinstance(kwargs["input"][0], dict) or isinstance(kwargs["input"][0], str)) + and ( + isinstance(kwargs["input"][0], dict) + or isinstance(kwargs["input"][0], str) + ) ) or isinstance(kwargs["input"], (dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert isinstance( @@ -13049,7 +12998,9 @@ class CompletionCustomHandler( assert response_obj == None ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) + assert isinstance(kwargs["messages"], list) and isinstance( + kwargs["messages"][0], dict + ) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) @@ -13057,12 +13008,15 @@ class CompletionCustomHandler( assert isinstance(kwargs["start_time"], (datetime, type(None))) assert isinstance(kwargs["stream"], bool) assert isinstance(kwargs["user"], (str, type(None))) - assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance( - kwargs["input"], (dict, str) - ) + assert ( + isinstance(kwargs["input"], list) + and isinstance(kwargs["input"][0], dict) + ) or isinstance(kwargs["input"], (dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) + isinstance( + kwargs["original_response"], (str, litellm.CustomStreamWrapper) + ) or kwargs["original_response"] == None ) assert isinstance(kwargs["additional_args"], (dict, type(None))) @@ -13080,7 +13034,9 @@ class CompletionCustomHandler( assert isinstance(messages, list) and isinstance(messages[0], dict) ## KWARGS assert isinstance(kwargs["model"], str) - assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict) + assert isinstance(kwargs["messages"], list) and isinstance( + kwargs["messages"][0], dict + ) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) assert isinstance(kwargs["start_time"], (datetime, type(None))) @@ -13092,7 +13048,9 @@ class CompletionCustomHandler( async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): try: - print("in async_log_success_event", kwargs, response_obj, start_time, end_time) + print( + "in async_log_success_event", kwargs, response_obj, start_time, end_time + ) self.states.append("async_success") ## START TIME assert isinstance(start_time, datetime) @@ -13121,7 +13079,9 @@ class CompletionCustomHandler( assert isinstance(kwargs["input"], (list, dict, str)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) + isinstance( + kwargs["original_response"], (str, litellm.CustomStreamWrapper) + ) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) ) @@ -13153,7 +13113,9 @@ class CompletionCustomHandler( assert isinstance(kwargs["input"], (list, str, dict)) assert isinstance(kwargs["api_key"], (str, type(None))) assert ( - isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper)) + isinstance( + kwargs["original_response"], (str, litellm.CustomStreamWrapper) + ) or inspect.isasyncgen(kwargs["original_response"]) or inspect.iscoroutine(kwargs["original_response"]) or kwargs["original_response"] == None diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index d14f236f765..071f216fd16 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -54,6 +54,7 @@ def _provider_fields() -> dict[str, object]: "thinking_blocks": [{"type": "thinking", "thinking": SECRET}], "reasoningContentBlocks": [{"reasoningText": {"text": SECRET}}], "citations": [{"cited_text": SECRET}], + "compaction_blocks": [{"type": "compaction", "content": SECRET, "signature": "compaction-signature"}], "thought_signatures": [SIGNATURE], } diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index f867ef6f6aa..a1be6e5bbfe 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4529,7 +4529,7 @@ _SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so @pytest.mark.parametrize( "provider,expected", - [("anthropic", True), ("vertex_ai", True), ("azure_ai", False), ("bedrock", False), ("bedrock_converse", False)], + [("anthropic", True), ("vertex_ai", True), ("azure_ai", True), ("bedrock", False), ("bedrock_converse", False)], ) @pytest.mark.parametrize( "optional_params,messages", @@ -4547,7 +4547,7 @@ def test_requires_native_compaction_beta_per_provider( assert requires_native_compaction_beta(provider, optional_params, messages) is expected -@pytest.mark.parametrize("provider", ["anthropic", "vertex_ai"]) +@pytest.mark.parametrize("provider", ["anthropic", "azure_ai", "vertex_ai"]) @pytest.mark.parametrize( "optional_params,messages", [ diff --git a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py index 4913d2b0d1d..0194e6736d5 100644 --- a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py +++ b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py @@ -215,3 +215,36 @@ async def test_count_uses_the_shared_api_base_chat_would_use(monkeypatch: pytest assert result is not None assert result.total_tokens == 6 assert route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("assistant_content", "expected_beta"), + [ + ( + [{"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"}], + "compact-2026-09-04,token-counting-2024-11-01", + ), + ("Use Recipe and Step tables.", "token-counting-2024-11-01"), + ], + ids=("signed_compaction_block", "plain_turn"), +) +async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_compaction_block( + monkeypatch: pytest.MonkeyPatch, assistant_content: object, expected_beta: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 376})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-5-5", + messages=[ + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": "What tables did we pick?"}, + ], + contents=None, + deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}}, + request_model="azure_ai/claude-sonnet-5-5", + ) + assert result is not None + assert result.total_tokens == 376 + assert route.calls.last.request.headers["anthropic-beta"] == expected_beta diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 38483b8a1fa..6eaf11392b3 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -596,3 +596,49 @@ async def test_outgoing_azure_request_keeps_an_object_output_format_under_drop_p body: Final = await _azure_request_body_for_output_format(output_format, drop_params=True) assert body["output_format"] == output_format + + +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig"} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +@pytest.mark.parametrize( + ("messages", "params"), + [ + ([{"role": "user", "content": "Summarize the chat so far."}], {"compaction": {"type": "summarize"}}), + ( + [{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "Go on"}], + {}, + ), + ], + ids=["compaction_param", "signed_compaction_block"], +) +async def test_outgoing_azure_compaction_request_carries_the_compact_2026_09_04_beta( + messages: list[object], params: dict[str, object] +) -> None: + """Foundry answers 400 "compaction: this parameter requires anthropic-beta: compact-2026-09-04" for the param + and 400 "Input tag 'compaction' ... does not match" for a replayed block without the header, and 200 with it + (live, claude-sonnet-5-5 on Azure AI Foundry, 2026-10-20).""" + received: Final[list[httpx.Request]] = [] + + def foundry_answers(request: httpx.Request) -> httpx.Response: + received.append(request) + return _foundry_answer(request) + + upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers)) + try: + await litellm.anthropic.messages.acreate( + max_tokens=256, + messages=messages, + model="azure_ai/claude-sonnet-5-5", + api_key="test-api-key", + api_base="https://test-resource.services.ai.azure.com", + client=upstream, + **params, + ) + finally: + await upstream.close() + + assert len(received) == 1, received + assert received[0].headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1, dict(received[0].headers) diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py index 1179b0dd5da..eee31de4533 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py @@ -5,10 +5,12 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) +from typing import Final from unittest.mock import MagicMock, patch import pytest +from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig from litellm.types.router import GenericLiteLLMParams @@ -484,3 +486,31 @@ def test_chat_flagged_model_keeps_mid_conversation_system_role_in_place(local_mo "role": "system", "content": [{"type": "text", "text": "Answer with exactly one word."}], } + + +@pytest.mark.parametrize( + ("optional_params", "messages", "expected"), + [ + ({"compaction": {"type": "summarize"}}, [{"role": "user", "content": "Hello"}], True), + ( + {}, + [ + {"role": "assistant", "content": [{"type": "compaction", "content": "Summary.", "signature": "sig"}]}, + {"role": "user", "content": "Go on"}, + ], + True, + ), + ({}, [{"role": "user", "content": "Hello"}], False), + ], + ids=["compaction_param", "signed_compaction_block", "no_compaction"], +) +def test_azure_chat_compaction_sends_the_compact_2026_09_04_beta( + local_beta_headers_config: None, optional_params: dict[str, object], messages: list[object], expected: bool +) -> None: + headers: Final = AzureAnthropicConfig().update_headers_with_optional_anthropic_beta( + headers={}, optional_params=optional_params, messages=messages + ) + + filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="azure_ai") + + assert ("compact-2026-09-04" in filtered.get("anthropic-beta", "").split(",")) is expected diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 67a74d0fb4f..3944310b9dd 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1307,21 +1307,29 @@ def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_head @pytest.mark.parametrize( - "client_beta,header", + "client_beta,body,header", [ - ("context-1m-2025-08-07", "context-management-2025-06-27"), - ("inline-tools-2026-09-15", "context-management-2025-06-27,inline-tools-2026-09-15"), + ( + "context-1m-2025-08-07", + ["context-1m-2025-08-07", "context-management-2025-06-27"], + "context-management-2025-06-27", + ), + ( + "inline-tools-2026-09-15", + ["context-management-2025-06-27"], + "context-management-2025-06-27,inline-tools-2026-09-15", + ), ], ids=["body_beta", "header_only_beta"], ) -def test_vertex_ai_anthropic_litellm_betas_replace_the_client_body_betas( - local_beta_headers_config: None, client_beta: str, header: str +def test_vertex_ai_anthropic_litellm_betas_merge_with_the_client_body_betas( + local_beta_headers_config: None, client_beta: str, body: list[str], header: str ) -> None: headers: Final[dict[str, str]] = {} result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers) - assert result["anthropic_beta"] == ["context-management-2025-06-27"] + assert result["anthropic_beta"] == body assert headers["anthropic-beta"] == header diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index da12c450aa4..9e5beac637f 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -83,7 +83,7 @@ class TestAnthropicBetaHeadersFiltering: filtered = filter_and_transform_beta_headers( beta_headers=all_headers, provider=provider ) - assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "vertex_ai")) + assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "azure_ai", "vertex_ai")) for header in unsupported_headers: assert ( From fbbcd524390fbe111be6ed2347020d8a84a34ac1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:27:48 -0700 Subject: [PATCH 10/12] fix(vertex_ai,azure_ai): auto-add thinking display and tool change betas, redact streamed compaction summaries Folds the header changes from #44103 into this PR. Vertex AI and Azure AI Claude routes now add thinking-display-updates-2026-08-18 when thinking.display is "updates" and mid-conversation-tool-changes-2026-07-01 when a tool_addition block is present, matching the Anthropic and Bedrock routes. With turn_off_message_logging on, compaction summaries from streamed messages and streamed chat completions are now redacted in spend logs and callback payloads, like the non-streamed response already was. --- .../src/providers/anthropic/beta.rs | 8 +- litellm/anthropic_beta_headers_config.json | 2 + litellm/litellm_core_utils/redact_messages.py | 9 ++- litellm/llms/anthropic/common_utils.py | 16 ++-- .../llms/azure_ai/anthropic/transformation.py | 14 +++- .../transformation.py | 20 ++++- .../anthropic/transformation.py | 3 + litellm/types/llms/anthropic.py | 5 +- .../test_litellm_logging.py | 17 ++++- .../test_redact_messages.py | 2 + .../anthropic/test_anthropic_common_utils.py | 14 ++-- ...azure_anthropic_messages_transformation.py | 42 ++++++++++ .../test_azure_anthropic_transformation.py | 54 +++++++++++++ ...artner_models_anthropic_messages_config.py | 76 +++++++++++++++++++ ...partner_models_anthropic_transformation.py | 48 ++++++++++++ .../test_anthropic_beta_headers_filtering.py | 1 + 16 files changed, 309 insertions(+), 22 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index e731a749026..d00bb467fa8 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -258,14 +258,18 @@ impl AnthropicBeta { Self::InterleavedThinking20250514 | Self::WebSearch20250305 => { &[BetaProvider::BedrockConverse, BetaProvider::Bedrock] } - Self::MidConversationOutputConfig20260701 - | Self::MidConversationToolChanges20260701 => &[ + Self::MidConversationOutputConfig20260701 => &[ BetaProvider::AzureAi, BetaProvider::BedrockConverse, BetaProvider::BedrockMantle, BetaProvider::VertexAi, BetaProvider::Databricks, ], + Self::MidConversationToolChanges20260701 => &[ + BetaProvider::BedrockConverse, + BetaProvider::BedrockMantle, + BetaProvider::Databricks, + ], Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[ BetaProvider::AzureAi, BetaProvider::BedrockConverse, diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 8cd6937510b..eaa6b90456b 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -60,6 +60,7 @@ "mcp-client-2025-11-20": "mcp-client-2025-11-20", "mcp-client-2025-04-04": "mcp-client-2025-04-04", "mcp-servers-2025-12-04": null, + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, "per-turn-control-2026-07-01": "per-turn-control-2026-07-01", @@ -204,6 +205,7 @@ "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, "mcp-servers-2025-12-04": null, + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, "per-turn-control-2026-07-01": "per-turn-control-2026-07-01", diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ec42c0e059a..fb01e9d42d5 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -149,7 +149,14 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[ **({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}), **{ key: None - for key in ("thinking_blocks", "reasoningContentBlocks", "citations", "compaction_blocks") + for key in ( + "thinking_blocks", + "reasoningContentBlocks", + "citations", + "compaction_blocks", + "compaction_start", + "compaction_delta", + ) if key in fields }, } diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 26e5dab890e..7cba8bd47f4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -52,6 +52,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, + LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, @@ -548,10 +549,12 @@ class AnthropicModelInfo(BaseLLMModelInfo): file_ids: Final = get_file_ids_from_messages(messages) return len(file_ids) > 0 - def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool: - if not isinstance(thinking, dict): - return False - return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates" + def is_thinking_display_updates_used(self, thinking: object) -> bool: + match thinking: + case {"type": "adaptive" | "enabled", "display": "updates"}: + return True + case _: + return False def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool: for message in messages: @@ -1003,7 +1006,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params) display: Final = thinking.get("display") - supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {} + keeps_display: Final = display in ("summarized", "omitted") or ( + display == "updates" and custom_llm_provider in LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS + ) + supported_display: Final = {"display": display} if keeps_display else {} carried_keys: Final = carried_thinking_keys( thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params ) diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index 18a4eeb7c0a..2bf6b57f7ed 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -2,13 +2,18 @@ Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication """ +from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter + from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams +_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object]) + def _promote_extra_body_to_optional_params(optional_params: dict) -> None: """Promote anthropic-native passthrough keys out of ``extra_body``. @@ -96,9 +101,16 @@ class AzureAnthropicConfig(AnthropicConfig): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + is_thinking_display_updates_used=self.is_thinking_display_updates_used( + _OPTIONAL_PARAMS.validate_python(optional_params).get("thinking") + ), ) # Merge headers - Azure auth (api-key or Authorization) takes precedence - headers = {**anthropic_headers, **headers} + litellm_beta: Final = _OPTIONAL_PARAMS.validate_python(anthropic_headers).get("anthropic-beta") + litellm_betas: Final[Mapping[str, str]] = ( + {"anthropic-beta": litellm_beta} if isinstance(litellm_beta, str) else {} + ) + headers = {**anthropic_headers, **headers, **litellm_betas} # Ensure anthropic-version header is set if "anthropic-version" not in headers: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 18edc20f942..4324f944651 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -11,6 +11,8 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( from litellm.types.llms.anthropic import ( ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ) from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.llms.vertex_ai import VertexPartnerProvider @@ -120,11 +122,23 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) - if requires_native_compaction_beta("vertex_ai", _OPTIONAL_PARAMS.validate_python(optional_params), messages): + typed_params: Final = _OPTIONAL_PARAMS.validate_python(optional_params) + if requires_native_compaction_beta("vertex_ai", typed_params, messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) - if beta_values: - headers["anthropic-beta"] = ",".join(beta_values) + thinking_display_betas: Final = ( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) + if anthropic_model_info.is_thinking_display_updates_used(typed_params.get("thinking")) + else () + ) + tool_change_betas: Final = ( + (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) + if anthropic_model_info.is_mid_conversation_tool_change_used(messages) + else () + ) + all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas) + if all_beta_values: + headers["anthropic-beta"] = ",".join(sorted(all_beta_values)) return headers, api_base diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index bfdca373803..e7aea7d0057 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -156,6 +156,9 @@ class VertexAIAnthropicConfig(AnthropicConfig): file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="vertex_ai", + is_thinking_display_updates_used=self.is_thinking_display_updates_used( + _NON_DEFAULT_PARAMS.validate_python(data).get("thinking") + ), ) beta_set: Final = set(auto_betas) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 55c383a25a7..e13df3aaa31 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -702,7 +702,7 @@ class AnthropicResponseContentBlockCompaction(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) type: Literal["compaction"] - content: str | None + content: str | None = None signature: str | None = None @@ -783,6 +783,9 @@ class AnthropicThinkingParam(TypedDict, total=False): block_binding: ReadOnly[AnthropicThinkingBlockBinding] +LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("azure_ai", "vertex_ai")) + + class ANTHROPIC_HOSTED_TOOLS(str, Enum): WEB_SEARCH = "web_search" BASH = "bash" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 96d9edb90d9..15ea822147c 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6512,10 +6512,19 @@ def test_handle_anthropic_messages_response_logging_reparses_the_served_body_wit assert logged.usage.completion_tokens == 7 # type: ignore[attr-defined] -def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction_response(): +@pytest.mark.parametrize( + "compaction_block", + [ + {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"}, + {"type": "compaction", "signature": "sig-abc"}, + ], + ids=["with_summary", "without_content"], +) +def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction_response( + compaction_block: dict[str, str], +): logging_obj = _anthropic_messages_logging_obj() logging_obj.optional_params = {} - compaction_block = {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"} logged = logging_obj._handle_anthropic_messages_response_logging( result={ @@ -6534,7 +6543,9 @@ def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction assert logged.id == "msg_compacted" assert logged.usage.prompt_tokens == 410 # type: ignore[attr-defined] assert logged.usage.completion_tokens == 95 # type: ignore[attr-defined] - assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block] # type: ignore[union-attr] + assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [ # type: ignore[union-attr] + {"content": None, **compaction_block} + ] def test_handle_anthropic_messages_response_logging_passes_model_response_through(): diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 071f216fd16..a81eeedb3c2 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -55,6 +55,8 @@ def _provider_fields() -> dict[str, object]: "reasoningContentBlocks": [{"reasoningText": {"text": SECRET}}], "citations": [{"cited_text": SECRET}], "compaction_blocks": [{"type": "compaction", "content": SECRET, "signature": "compaction-signature"}], + "compaction_start": {"type": "compaction", "content": SECRET}, + "compaction_delta": {"type": "compaction_delta", "content": SECRET}, "thought_signatures": [SIGNATURE], } diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index a1be6e5bbfe..6f84e03fb62 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4236,15 +4236,17 @@ def test_thinking_display_beta_requires_active_thinking(thinking: object, expect @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - ("display", "expected_thinking"), + ("provider", "display", "expected_thinking"), ( - ("summarized", {"type": "adaptive", "display": "summarized"}), - ("omitted", {"type": "adaptive", "display": "omitted"}), - ("updates", {"type": "adaptive"}), + ("azure_ai", "summarized", {"type": "adaptive", "display": "summarized"}), + ("azure_ai", "omitted", {"type": "adaptive", "display": "omitted"}), + ("azure_ai", "updates", {"type": "adaptive", "display": "updates"}), + ("vertex_ai", "updates", {"type": "adaptive", "display": "updates"}), + ("anthropic", "updates", {"type": "adaptive"}), ), ) def test_shared_legacy_thinking_translation_preserves_supported_display( - display: str, expected_thinking: dict[str, str] + provider: str, display: str, expected_thinking: dict[str, str] ) -> None: from litellm.llms.anthropic.common_utils import AnthropicModelInfo @@ -4255,7 +4257,7 @@ def test_shared_legacy_thinking_translation_preserves_supported_display( AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( model="claude-opus-5", optional_params=optional_params, - custom_llm_provider="azure_ai", + custom_llm_provider=provider, drop_params=False, ) diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 6eaf11392b3..8a3a5ab8a3a 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -642,3 +642,45 @@ async def test_outgoing_azure_compaction_request_carries_the_compact_2026_09_04_ assert len(received) == 1, received assert received[0].headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1, dict(received[0].headers) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("feature", ("thinking", "output_config", "tool_addition", "tool_removal", "none")) +@pytest.mark.parametrize("explicit_beta", (False, True)) +def test_messages_feature_beta_headers(feature: str, explicit_beta: bool) -> None: + from typing import Final + + from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig + from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ) + + beta: Final = ( + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + if feature == "thinking" + else ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value + if feature == "output_config" + else ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER + ) + message: Final = ( + {"role": "system", "content": [], "output_config": {"effort": "high"}} + if feature == "output_config" + else {"role": "system", "content": [{"type": feature, "tool": {"type": "tool_reference", "name": "ping"}}]} + if feature in ("tool_addition", "tool_removal") + else {"role": "user", "content": "Reply with OK"} + ) + headers, _ = AzureAnthropicMessagesConfig().validate_anthropic_messages_environment( + headers={"anthropic-beta": f"existing-beta,{beta}" if explicit_beta else "existing-beta"}, + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Hello"}, message], + optional_params={"thinking": {"type": "adaptive", "display": "updates"}} if feature == "thinking" else {}, + litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"}, + api_key="test-key", + api_base="https://example.com/anthropic", + ) + + betas: Final = headers["anthropic-beta"].split(",") + assert betas.count(beta) == int(feature != "none" or explicit_beta) + assert betas.count("existing-beta") == 1 diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py index eee31de4533..6ff2a6973a3 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py @@ -514,3 +514,57 @@ def test_azure_chat_compaction_sends_the_compact_2026_09_04_beta( filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="azure_ai") assert ("compact-2026-09-04" in filtered.get("anthropic-beta", "").split(",")) is expected + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("thinking_type", ("adaptive", "enabled")) +@pytest.mark.parametrize("display", ("summarized", "omitted", "updates")) +def test_chat_preserves_thinking_display_and_beta(thinking_type: str, display: str) -> None: + from typing import Final + + from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + config: Final = AzureAnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={"thinking": {"type": thinking_type, "display": display, "budget_tokens": 2048}}, + optional_params={}, + model="claude-fable-5-1", + drop_params=False, + ) + headers: Final = config.validate_environment( + headers={"anthropic-beta": "existing-beta"}, + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + api_key="test-key", + ) + result: Final = config.transform_request( + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + + assert result["thinking"]["type"] == "adaptive" + assert result["thinking"]["display"] == display + assert (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers["anthropic-beta"].split(",")) == (display == "updates") + assert "existing-beta" in headers["anthropic-beta"].split(",") + + +@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"})) +def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None: + from typing import Final + + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + params: Final = {"thinking": thinking} + headers: Final = AzureAnthropicConfig().validate_environment( + headers={}, model="claude-fable-5-1", messages=[], optional_params=params, + litellm_params={}, api_key="test-key", + ) + + assert params["thinking"] == thinking + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 3bf73debf65..d1d5351aeb8 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -918,3 +918,79 @@ async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 assert sent_body["compaction"] == {"type": "summarize"} assert "anthropic_beta" not in sent_body + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("feature", ("thinking", "output_config", "tool_addition", "tool_removal", "none")) +@pytest.mark.parametrize("explicit_beta", (False, True)) +def test_messages_feature_beta_headers(feature: str, explicit_beta: bool) -> None: + from typing import Final + from unittest.mock import patch + + from google.oauth2.credentials import Credentials + from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import VertexAIPartnerModelsAnthropicMessagesConfig + from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ) + + beta: Final = ( + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + if feature == "thinking" + else ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value + if feature == "output_config" + else ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER + ) + message: Final = ( + {"role": "system", "content": [], "output_config": {"effort": "high"}} + if feature == "output_config" + else {"role": "system", "content": [{"type": feature, "tool": {"type": "tool_reference", "name": "ping"}}]} + if feature in ("tool_addition", "tool_removal") + else {"role": "user", "content": "Reply with OK"} + ) + with patch( + "google.auth.default", + return_value=(MagicMock(spec=Credentials, token="test-token", expired=False), "test-project"), + ): + headers, _ = VertexAIPartnerModelsAnthropicMessagesConfig().validate_anthropic_messages_environment( + headers={"anthropic-beta": f"existing-beta,{beta}" if explicit_beta else "existing-beta"}, + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Hello"}, message], + optional_params={"thinking": {"type": "adaptive", "display": "updates"}} if feature == "thinking" else {}, + litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"}, + api_key="test-key", + api_base="https://example.com/anthropic", + ) + + betas: Final = headers["anthropic-beta"].split(",") + assert betas.count(beta) == int(feature != "none" or explicit_beta) + assert betas.count("existing-beta") == 1 + + if feature in ("tool_addition", "tool_removal"): + filtered_headers: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") + assert filtered_headers.get("anthropic-beta", "").split(",").count(beta) == 1 + + +@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"})) +def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None: + from typing import Final + from unittest.mock import patch + + from google.oauth2.credentials import Credentials + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + params: Final = {"thinking": thinking} + with patch( + "google.auth.default", + return_value=(MagicMock(spec=Credentials, token="test-token", expired=False), "test-project"), + ): + headers, _ = VertexAIPartnerModelsAnthropicMessagesConfig().validate_anthropic_messages_environment( + headers={}, model="claude-fable-5-1", messages=[], optional_params=params, + litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"}, + api_key="test-key", api_base="https://example.com/anthropic", + ) + + assert params["thinking"] == thinking + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 3944310b9dd..462b790c606 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1431,3 +1431,51 @@ def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wir assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 assert sent_body["compaction"] == {"type": "summarize"} assert "anthropic_beta" not in sent_body + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("thinking_type", ("adaptive", "enabled")) +@pytest.mark.parametrize("display", ("summarized", "omitted", "updates")) +def test_chat_preserves_thinking_display_and_beta(thinking_type: str, display: str) -> None: + from typing import Final + + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + config: Final = VertexAIAnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={"thinking": {"type": thinking_type, "display": display, "budget_tokens": 2048}}, + optional_params={}, + model="claude-fable-5-1", + drop_params=False, + ) + optional_params["extra_headers"] = {"anthropic-beta": "existing-beta"} + headers: Final = {} + result: Final = config.transform_request( + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + + assert result["thinking"]["type"] == "adaptive" + assert result["thinking"]["display"] == display + assert (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers["anthropic-beta"].split(",")) == (display == "updates") + assert "existing-beta" in headers["anthropic-beta"].split(",") + + +@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"})) +def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None: + from typing import Final + + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + headers: Final = {} + result: Final = VertexAIAnthropicConfig().transform_request( + model="claude-fable-5-1", messages=[{"role": "user", "content": "Hello"}], + optional_params={"thinking": thinking}, litellm_params={}, headers=headers, + ) + + assert result["thinking"] == thinking + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index 9e5beac637f..d2741ecb897 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -84,6 +84,7 @@ class TestAnthropicBetaHeadersFiltering: beta_headers=all_headers, provider=provider ) assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "azure_ai", "vertex_ai")) + assert ("mid-conversation-tool-changes-2026-07-01" in filtered) is (provider != "bedrock_converse") for header in unsupported_headers: assert ( From b649f33f17700ba48fbd9ad04b8b831f166cf4df Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:14:32 -0700 Subject: [PATCH 11/12] fix(vertex_ai,azure_ai,rust): add tool change betas on count tokens and Rust messages, bill Rust server tool replies Vertex and Azure count tokens now add mid-conversation-tool-changes-2026-07-01 when the history carries a tool_addition or tool_removal block, through one shared helper with the compaction beta. The Rust /v1/messages route adds thinking-display-updates-2026-08-18 and mid-conversation-tool-changes-2026-07-01 when a request uses them, on the providers that accept each beta. Rust-route logging parses the served body with the Anthropic response parser, so web_search, pause_turn, mcp_tool_use and container_upload replies get a spend row. Legacy enabled thinking keeps display updates on anthropic too, and the Vertex transform reads drop_params through drops_unsupported_params. --- .../src/anthropic/messages/transformation.rs | 99 ++++++++++++++++++- .../src/vertex_ai/messages/transformation.rs | 11 +++ litellm/litellm_core_utils/litellm_logging.py | 11 +-- litellm/llms/anthropic/common_utils.py | 24 ++++- .../anthropic/count_tokens/transformation.py | 8 +- .../anthropic/transformation.py | 3 +- .../count_tokens/handler.py | 10 +- litellm/types/llms/anthropic.py | 2 +- .../test_litellm_logging.py | 63 +++++++++++- .../anthropic/test_anthropic_common_utils.py | 3 +- .../count_tokens/test_token_counter.py | 20 ++++ .../count_tokens/test_handler.py | 26 +++++ 12 files changed, 251 insertions(+), 29 deletions(-) diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 6657cbf262f..2351df0fac1 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -1,7 +1,8 @@ use litellm_auth::CredentialPlacement; use litellm_llms_types::{ formats::messages::{ - ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed, + ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message, + MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay, }, providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS}, recognized::Recognized, @@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet { .then_some(AnthropicBeta::FastMode20260201), messages_carry_output_config(&request.messages) .then_some(AnthropicBeta::PerTurnControl20260701), + uses_thinking_display_updates(params) + .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), + messages_carry_tool_changes(&request.messages) + .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120), ] @@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool { .any(|message| message.extra.contains_key("output_config")) } +fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool { + let display = match params.thinking.as_ref().and_then(Recognized::known) { + Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(), + Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(), + _ => None, + }; + display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates) +} + +fn is_tool_reference_change(block: &ContentBlock) -> bool { + matches!( + &block.block_type, + Some(ContentBlockType::Other(block_type)) + if block_type == "tool_addition" || block_type == "tool_removal" + ) && block + .extra + .get("tool") + .and_then(|tool| tool.get("type")) + .and_then(Value::as_str) + == Some("tool_reference") +} + +fn messages_carry_tool_changes(messages: &[Message]) -> bool { + messages + .iter() + .filter(|message| message.role == "system") + .flat_map(Message::blocks) + .any(is_tool_reference_change) +} + fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error { Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter { model: model.into(), @@ -1071,6 +1106,41 @@ mod tests { json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}), &[] )] + #[case::adaptive_display_updates( + json!({"thinking": {"type": "adaptive", "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::enabled_display_updates( + json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])] + #[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])] + #[case::system_tool_addition( + json!({"messages": [ + {"role": "user", "content": "Use the ping tool"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] + #[case::system_tool_removal( + json!({"messages": [ + {"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] + #[case::user_tool_addition( + json!({"messages": [ + {"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &[] + )] + #[case::tool_addition_without_a_tool_reference( + json!({"messages": [ + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]}, + ]}), + &[] + )] fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { assert_eq!(feature_betas(&request(fields)), betas(expected)); } @@ -1093,6 +1163,33 @@ mod tests { ); } + #[rstest] + #[case::vertex_ai( + BetaProvider::VertexAi, + &["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"] + )] + #[case::azure_ai( + BetaProvider::AzureAi, + &["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"] + )] + #[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])] + #[case::databricks(BetaProvider::Databricks, &[])] + fn tool_change_and_display_betas_reach_the_providers_that_accept_them( + #[case] provider: BetaProvider, + #[case] expected: &[&str], + ) { + let fields = json!({ + "thinking": {"type": "adaptive", "display": "updates"}, + "messages": [ + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + }); + assert_eq!( + provider_feature_betas(&request(fields), provider), + betas(expected) + ); + } + #[rstest] fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() { let client = headers(&[("anthropic-beta", "claude-code-20250219")]); diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index f4e8ab3bcca..775cf005d4c 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -662,6 +662,17 @@ mod tests { &["compact-2026-01-12", "context-management-2025-06-27"] )] #[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])] + #[case::thinking_display_updates( + json!({"thinking": {"type": "adaptive", "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::tool_addition( + json!({"messages": [ + {"role": "user", "content": "Use the ping tool"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]} + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] #[case::advisor_tool_is_not_a_vertex_beta( json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}), &[] diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e2eed3cf6d5..7b0377ec4df 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -551,6 +551,7 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) +_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) def _provider_response_id(source: object) -> str | None: @@ -4693,13 +4694,11 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one") return litellm.ModelResponse(model=self.model) else: - from litellm.types.llms.anthropic import AnthropicResponse - - pydantic_result: Final = AnthropicResponse.model_validate(result) - import httpx - + served_body: Final = _SERVED_ANTHROPIC_BODY.validate_python( + result.model_dump() if isinstance(result, BaseModel) else result + ) result = litellm.AnthropicConfig().transform_parsed_response( - completion_response=pydantic_result.model_dump(), + completion_response=served_body, raw_response=httpx.Response( status_code=200, headers={}, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 7cba8bd47f4..fc3314baeff 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -44,6 +44,7 @@ from litellm.llms.anthropic.wif import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, @@ -150,6 +151,27 @@ def requires_native_compaction_beta( ) +def count_tokens_feature_betas( + custom_llm_provider: str, + optional_params: Mapping[str, object], + messages: Sequence[object], +) -> tuple[str, ...]: + return tuple( + beta + for beta, used in ( + ( + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value, + requires_native_compaction_beta(custom_llm_provider, optional_params, messages), + ), + ( + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + AnthropicModelInfo().is_mid_conversation_tool_change_used(messages), + ), + ) + if used + ) + + def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching @@ -1313,8 +1335,6 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work if web_search_tool_used: - from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(all_betas) > 0: headers["anthropic-beta"] = ",".join(all_betas) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py index 0e42de201d8..0c661514123 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py @@ -8,12 +8,11 @@ from collections.abc import Sequence from typing import Final from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION -from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers, requires_native_compaction_beta +from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) from litellm.llms.azure.common_utils import BaseAzureLLM -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.router import GenericLiteLLMParams @@ -49,10 +48,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): "Content-Type": "application/json", "anthropic-version": "2023-06-01", "anthropic-beta": merge_anthropic_beta_headers( - ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, - ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value - if requires_native_compaction_beta("azure_ai", {}, messages) - else None, + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages) ), **({"x-api-key": api_key} if api_key else {}), } diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index e7aea7d0057..5fa24bd1ec1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -7,6 +7,7 @@ import httpx from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES @@ -176,7 +177,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"])) else frozenset[str]() ) - drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {}) data.pop("extra_headers", None) injected_betas: Final = frozenset(beta_set) | _client_betas( diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 103c51eb71a..94d2e130c0f 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -11,11 +11,10 @@ from typing import Any, Final from pydantic import TypeAdapter -from litellm.llms.anthropic.common_utils import requires_native_compaction_beta +from litellm.llms.anthropic.common_utils import count_tokens_feature_betas from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES _COUNT_TOKENS_PUBLISHERS: Final = ( ("claude", "anthropic"), @@ -33,11 +32,10 @@ def count_tokens_publisher(model: str) -> str | None: def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]: authorization: Final = {"Authorization": f"Bearer {access_token}"} - if not requires_native_compaction_beta( + betas: Final = count_tokens_feature_betas( "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"]) - ): - return authorization - return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value} + ) + return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization class VertexAIPartnerModelsTokenCounter(VertexBase): diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index e13df3aaa31..c3cf532db09 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -783,7 +783,7 @@ class AnthropicThinkingParam(TypedDict, total=False): block_binding: ReadOnly[AnthropicThinkingBlockBinding] -LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("azure_ai", "vertex_ai")) +LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("anthropic", "azure_ai", "vertex_ai")) class ANTHROPIC_HOSTED_TOOLS(str, Enum): diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 15ea822147c..873ef0d0baf 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6541,11 +6541,64 @@ def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction assert isinstance(logged, ModelResponse) assert logged.id == "msg_compacted" - assert logged.usage.prompt_tokens == 410 # type: ignore[attr-defined] - assert logged.usage.completion_tokens == 95 # type: ignore[attr-defined] - assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [ # type: ignore[union-attr] - {"content": None, **compaction_block} - ] + assert logged.usage.prompt_tokens == 410 + assert logged.usage.completion_tokens == 95 + assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block] + + +_WEB_SEARCH_REPLY: Final = ( + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "weather"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "Weather"}], + }, + {"type": "text", "text": "It is sunny."}, +) + + +@pytest.mark.parametrize( + ("content", "stop_reason"), + [ + (list(_WEB_SEARCH_REPLY), "end_turn"), + (list(_WEB_SEARCH_REPLY), "pause_turn"), + ([{"type": "text", "text": "The context window is full."}], "model_context_window_exceeded"), + ( + [ + {"type": "mcp_tool_use", "id": "mcptoolu_1", "name": "lookup", "server_name": "docs", "input": {}}, + {"type": "mcp_tool_result", "tool_use_id": "mcptoolu_1", "is_error": False, "content": [{"type": "text", "text": "ok"}]}, + {"type": "text", "text": "Found it."}, + ], + "end_turn", + ), + ([{"type": "container_upload", "file_id": "file_1"}, {"type": "text", "text": "Uploaded."}], "end_turn"), + ], + ids=["web_search", "pause_turn", "model_context_window_exceeded", "mcp_tool_use", "container_upload"], +) +def test_handle_anthropic_messages_response_logging_bills_a_served_server_tool_response( + content: list[dict[str, object]], stop_reason: str +): + logging_obj = _anthropic_messages_logging_obj() + logging_obj.optional_params = {} + + logged = logging_obj._handle_anthropic_messages_response_logging( + result={ + "id": "msg_server_tools", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 1200, "output_tokens": 80}, + } + ) + + assert isinstance(logged, ModelResponse) + assert logged.id == "msg_server_tools" + assert logged.usage.prompt_tokens == 1200 + assert logged.usage.completion_tokens == 80 + assert logged.choices[0].message.content == content[-1]["text"] def test_handle_anthropic_messages_response_logging_passes_model_response_through(): diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 6f84e03fb62..0692a762bf8 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4242,7 +4242,8 @@ def test_thinking_display_beta_requires_active_thinking(thinking: object, expect ("azure_ai", "omitted", {"type": "adaptive", "display": "omitted"}), ("azure_ai", "updates", {"type": "adaptive", "display": "updates"}), ("vertex_ai", "updates", {"type": "adaptive", "display": "updates"}), - ("anthropic", "updates", {"type": "adaptive"}), + ("anthropic", "updates", {"type": "adaptive", "display": "updates"}), + ("databricks", "updates", {"type": "adaptive"}), ), ) def test_shared_legacy_thinking_translation_preserves_supported_display( diff --git a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py index 0194e6736d5..7807975be24 100644 --- a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py +++ b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py @@ -248,3 +248,23 @@ async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_comp assert result is not None assert result.total_tokens == 376 assert route.calls.last.request.headers["anthropic-beta"] == expected_beta + + +@pytest.mark.asyncio +async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 541})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-5-5", + messages=[ + {"role": "user", "content": "Use the ping tool once."}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + contents=None, + deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}}, + request_model="azure_ai/claude-sonnet-5-5", + ) + assert result is not None + assert result.total_tokens == 541 + assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01" diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py index b23461823f8..c03f08ab610 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py @@ -32,3 +32,29 @@ def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_dat ) def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None: assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"} + + +_TOOL_ADDITION: Final = { + "role": "system", + "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}], +} + + +@pytest.mark.parametrize( + ("request_data", "expected_beta"), + [ + ({"messages": [*_HISTORY, _TOOL_ADDITION]}, "mid-conversation-tool-changes-2026-07-01"), + ( + {"messages": [*_HISTORY, _TOOL_ADDITION], "compaction": {"type": "summarize"}}, + "compact-2026-09-04,mid-conversation-tool-changes-2026-07-01", + ), + ], + ids=["tool_addition", "tool_addition_with_compaction"], +) +def test_count_tokens_on_a_tool_change_request_sends_the_tool_change_beta( + request_data: dict[str, object], expected_beta: str +) -> None: + assert count_tokens_headers("token", request_data) == { + "Authorization": "Bearer token", + "anthropic-beta": expected_beta, + } From 3d7a8d0ba1c4d3d7cb34a9999bd6a6da44f102ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:49:50 -0700 Subject: [PATCH 12/12] style(rust): rewrap beta map test line with cargo fmt --- .../crates/llms-types/src/providers/anthropic/beta.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 921f083fbd6..8acd6a9b48c 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -520,8 +520,8 @@ mod tests { .and_then(serde_json::Value::as_str) .map(str::to_string) }; - let expected = cell(&format!("{provider}_header_only")) - .or_else(|| cell(provider.as_ref())); + let expected = + cell(&format!("{provider}_header_only")).or_else(|| cell(provider.as_ref())); let actual = beta.on(*provider).map(|name| name.to_string()); assert_eq!(actual, expected, "provider {provider}, beta {beta}"); if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)