From cdc280c86712662607a00ee1da53add5853cf5bd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:46:31 -0700 Subject: [PATCH] fix(vertex_ai): keep client body betas out of the Vertex body next to litellm betas, honor drop_params, and send the compaction beta on count_tokens Litellm's own betas replace a client's body anthropic_beta again, as on main, so a body beta Vertex rejects no longer turns a 200 into a 400. A malformed beta drops under drop_params and names its field otherwise. inline-tools-2026-09-15 joins compact-2026-09-04 as header-only, since Vertex rejects both in the body. Vertex count_tokens sends compact-2026-09-04 when the request carries compaction. --- .../anthropic/transformation.py | 42 ++-- .../count_tokens/handler.py | 20 +- litellm/types/llms/anthropic.py | 1 + ...artner_models_anthropic_messages_config.py | 23 ++- ...partner_models_anthropic_transformation.py | 181 +++++++++++------- .../count_tokens/test_handler.py | 34 ++++ 6 files changed, 206 insertions(+), 95 deletions(-) create mode 100644 tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 692a35f3a1f..18022f8d51d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -22,7 +22,9 @@ if TYPE_CHECKING: _NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object]) -_VERTEX_HEADER_ONLY_BETAS: Final = frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) +_VERTEX_HEADER_ONLY_BETAS: Final = frozenset( + {ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value, ANTHROPIC_BETA_HEADER_VALUES.INLINE_TOOLS_2026_09_15.value} +) _BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None) @@ -30,18 +32,27 @@ _BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str _REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object]) -def _beta_names(value: object) -> frozenset[str]: +def _beta_names(value: object) -> frozenset[str] | None: try: parsed: Final = _BETA_VALUE.validate_python(value) - except ValidationError as error: - raise AnthropicError( - status_code=400, - message=f"anthropic_beta must be a string or a list of strings, got {type(value).__name__}", - ) from error + except ValidationError: + return None names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or () return frozenset(name.strip() for name in names if name.strip()) +def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]: + names: Final = _beta_names(value) + if names is not None: + return names + if drop_params: + return frozenset() + raise AnthropicError( + status_code=400, + message=f"{field} must be a string or a list of strings, got {type(value).__name__}", + ) + + class VertexAIError(Exception): def __init__(self, status_code, message): self.status_code = status_code @@ -162,12 +173,17 @@ class VertexAIAnthropicConfig(AnthropicConfig): if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"])) else frozenset[str]() ) + drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {}) data.pop("extra_headers", None) - injected_betas: Final = frozenset(beta_set) | _beta_names(extra_headers.get("anthropic-beta")) - client_body_betas: Final = _beta_names(_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta")) + injected_betas: Final = frozenset(beta_set) | _client_betas( + extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params + ) + client_body_betas: Final = _client_betas( + _NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params + ) - body_betas: Final = (injected_betas | client_body_betas) - _VERTEX_HEADER_ONLY_BETAS + body_betas: Final = (injected_betas or client_body_betas) - _VERTEX_HEADER_ONLY_BETAS if body_betas: data["anthropic_beta"] = sorted(body_betas) elif "anthropic_beta" in data: @@ -176,7 +192,11 @@ class VertexAIAnthropicConfig(AnthropicConfig): injected_betas | compaction_betas | (client_body_betas & _VERTEX_HEADER_ONLY_BETAS) - | _beta_names(_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta")) + | _client_betas( + _NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"), + "The anthropic-beta header", + drop_params, + ) ) if header_betas: headers["anthropic-beta"] = ",".join(sorted(header_betas)) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 7100c9efde3..103c51eb71a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,11 +6,16 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Mapping, Sequence from typing import Any, Final +from pydantic import TypeAdapter + +from litellm.llms.anthropic.common_utils import requires_native_compaction_beta from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES _COUNT_TOKENS_PUBLISHERS: Final = ( ("claude", "anthropic"), @@ -19,10 +24,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( ) +_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object]) + + def count_tokens_publisher(model: str) -> str | None: return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None) +def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]: + authorization: Final = {"Authorization": f"Bearer {access_token}"} + if not requires_native_compaction_beta( + "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"]) + ): + return authorization + return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value} + + class VertexAIPartnerModelsTokenCounter(VertexBase): """ Token counter for Vertex AI Partner Models. @@ -157,8 +174,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): api_base=litellm_params.get("api_base"), ) - # Prepare headers - headers: Final = {"Authorization": f"Bearer {access_token}"} + headers: Final = dict(count_tokens_headers(access_token, request_data)) # Get async HTTP client from litellm import LlmProviders diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 6aeead7679f..9961ae12852 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -794,6 +794,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" COMPACT_2026_01_12 = "compact-2026-01-12" COMPACT_2026_09_04 = "compact-2026-09-04" + INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 6d3d1639615..3bf73debf65 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -132,7 +132,9 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out(): assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "") -def _validate_vertex_headers(client_headers, messages, optional_params=None): +def _validate_vertex_headers( + client_headers: dict[str, str], messages: list[object], optional_params: dict[str, object] | None = None +) -> dict[str, str]: config = VertexAIPartnerModelsAnthropicMessagesConfig() litellm_params = { "vertex_ai_project": "test-project", @@ -829,7 +831,7 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f assert missing == [] -_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"} +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} @pytest.mark.parametrize( @@ -849,22 +851,25 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", " ids=["compaction_param", "signed_compaction_block"], ) def test_native_compaction_reaches_vertex_with_compact_2026_09_04_beta( - local_beta_headers_config, client_headers, optional_params, messages -): + local_beta_headers_config: None, + client_headers: dict[str, str], + optional_params: dict[str, object], + messages: list[object], +) -> None: """Vertex serves on-demand compaction only behind compact-2026-09-04 (Google Cloud: beta on https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) and answers 400 "compaction: Extra inputs are not permitted" without it, so the beta rides along with the `compaction` param or a signed compaction block exactly once, whether or not the client sent it, and survives the Vertex beta filter.""" - headers = _validate_vertex_headers(client_headers, messages, optional_params) + headers: Final = _validate_vertex_headers(client_headers, messages, optional_params) - filtered = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") + filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") assert filtered["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 -def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out(): - headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) +def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None: + headers: Final = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") @@ -882,7 +887,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = { @pytest.mark.asyncio -async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config): +async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None: """A /v1/messages compaction request to a Vertex Claude deployment reaches Vertex with the beta in the anthropic-beta request header and never in the body, the shape Vertex accepts (live, 2026-10-05).""" mock_response: Final = MagicMock() diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index d90bb944116..67a74d0fb4f 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -430,8 +430,10 @@ def test_vertex_ai_anthropic_extra_headers_beta_propagation(): def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(local_beta_headers_config: None) -> None: """The chat path runs the Vertex beta filter over both the header and the `anthropic_beta` body field right - before the request goes out, so inline-tools-2026-09-15 must survive both (source and date in - tests/unit/test_anthropic_beta_headers_filtering.py).""" + before the request goes out, so inline-tools-2026-09-15 must survive it as the header (source and date in + tests/unit/test_anthropic_beta_headers_filtering.py) and stay out of the body field, which Vertex rejects + with 400 "Unexpected value(s) `inline-tools-2026-09-15` for the `anthropic_beta` parameter" (live, global, + claude-sonnet-5-5, 2026-10-20).""" config: Final = VertexAIAnthropicConfig() headers: Final[dict[str, str]] = {} optional_params: Final[dict[str, object]] = { @@ -452,7 +454,7 @@ def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(loc ) assert filtered_headers["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1 - assert "inline-tools-2026-09-15" in filtered_request["anthropic_beta"] + assert "inline-tools-2026-09-15" not in filtered_request.get("anthropic_beta", []) def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas(): @@ -1189,7 +1191,18 @@ def test_refusal_stop_details_kept_through_vertex_anthropic_transform(): assert provider_fields["stop_details"] == ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"] -_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"} +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} +_CLEAR_TOOL_USES: Final = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + +def _vertex_request(optional_params: dict[str, object], headers: dict[str, str], litellm_params: dict[str, object] | None = None) -> dict: + return VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"max_tokens": 100, "is_vertex_request": True, **optional_params}, + litellm_params=litellm_params or {}, + headers=headers, + ) @pytest.mark.parametrize( @@ -1214,18 +1227,17 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", " ids=["compaction_param", "signed_compaction_block_sent_back"], ) def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta( - local_beta_headers_config, optional_params, messages -): + local_beta_headers_config: None, optional_params: dict[str, object], messages: list[dict[str, object]] +) -> None: """The chat path skips the Anthropic beta injection for Vertex requests, so the Vertex config adds compact-2026-09-04 itself (Google Cloud: beta on https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) as the HTTP header only, since Vertex answered 400 "Unexpected value(s) `compact-2026-09-04` for the `anthropic_beta` parameter" with the name in the body field (us-east5, claude-sonnet-4-6, 2026-10-05), and the header must survive the Vertex beta filter.""" - config = VertexAIAnthropicConfig() - headers = {} + headers: Final[dict[str, str]] = {} - result = config.transform_request( + result: Final = VertexAIAnthropicConfig().transform_request( model="claude-opus-4-6", messages=messages, optional_params=optional_params, @@ -1240,111 +1252,134 @@ def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta( assert "compact-2026-09-04" not in filtered_body.get("anthropic_beta", []) -def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out(): - headers = {} +def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None: + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True}, - litellm_params={}, - headers=headers, - ) + result: Final = _vertex_request({}, headers) assert "compact-2026-09-04" not in result.get("anthropic_beta", []) assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") -def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(local_beta_headers_config): - headers = {} +def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field( + local_beta_headers_config: None, +) -> None: + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={ - "max_tokens": 100, - "is_vertex_request": True, + result: Final = _vertex_request( + { "compaction": {"type": "summarize"}, "extra_headers": {"anthropic-beta": "compact-2026-09-04,interleaved-thinking-2025-05-14"}, }, - litellm_params={}, - headers=headers, + headers, ) assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] -def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(local_beta_headers_config): - headers = {"anthropic-beta": "context-1m-2025-08-07"} +def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers( + local_beta_headers_config: None, +) -> None: + headers: Final = {"anthropic-beta": "context-1m-2025-08-07"} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}}, - litellm_params={}, - headers=headers, - ) + result: Final = _vertex_request({"compaction": {"type": "summarize"}}, headers) assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"] assert "anthropic_beta" not in result -def test_vertex_ai_anthropic_client_sent_body_compact_2026_09_04_moves_to_the_header(local_beta_headers_config): - """Vertex rejects `compact-2026-09-04` inside the body `anthropic_beta` field with 400 "Unexpected value(s)" - (live, 2026-10-05), so a client that opts in through that field still gets the beta as a header and keeps - its other betas in the body.""" - headers = {} +@pytest.mark.parametrize("header_only_beta", ["compact-2026-09-04", "inline-tools-2026-09-15"]) +def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_header( + local_beta_headers_config: None, header_only_beta: str +) -> None: + """Vertex rejects these betas inside the body `anthropic_beta` field with 400 "Unexpected value(s)" and + accepts them as the header (live, global, claude-sonnet-5-5, 2026-10-20), so a client that opts in through that field still + gets the beta as a header and keeps its other betas in the body.""" + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={ - "max_tokens": 100, - "is_vertex_request": True, - "compaction": {"type": "summarize"}, - "anthropic_beta": ["compact-2026-09-04", "interleaved-thinking-2025-05-14"], - }, - litellm_params={}, - headers=headers, + result: Final = _vertex_request( + {"anthropic_beta": [header_only_beta, "interleaved-thinking-2025-05-14"]}, headers ) - assert headers["anthropic-beta"] == "compact-2026-09-04" + assert headers["anthropic-beta"] == header_only_beta assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] +@pytest.mark.parametrize( + "client_beta,header", + [ + ("context-1m-2025-08-07", "context-management-2025-06-27"), + ("inline-tools-2026-09-15", "context-management-2025-06-27,inline-tools-2026-09-15"), + ], + ids=["body_beta", "header_only_beta"], +) +def test_vertex_ai_anthropic_litellm_betas_replace_the_client_body_betas( + local_beta_headers_config: None, client_beta: str, header: str +) -> None: + headers: Final[dict[str, str]] = {} + + result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers) + + assert result["anthropic_beta"] == ["context-management-2025-06-27"] + assert headers["anthropic-beta"] == header + + @pytest.mark.parametrize( "anthropic_beta", [["interleaved-thinking-2025-05-14"], ("interleaved-thinking-2025-05-14",)], ids=["list", "sdk_tuple"], ) -def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta): - headers = {} +def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta: object) -> None: + headers: Final[dict[str, str]] = {} - result = VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta}, - litellm_params={}, - headers=headers, - ) + result: Final = _vertex_request({"anthropic_beta": anthropic_beta}, headers) assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] assert "anthropic-beta" not in headers -@pytest.mark.parametrize("anthropic_beta", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"]) -def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400(anthropic_beta): +_MALFORMED_BETAS: Final = pytest.mark.parametrize( + "malformed", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"] +) +_MALFORMED_BETA_SOURCES: Final = pytest.mark.parametrize( + "source,field", + [("anthropic_beta", "anthropic_beta"), ("extra_headers", "The anthropic-beta header")], + ids=["body_field", "extra_header"], +) + + +def _malformed_beta_params(source: str, malformed: object) -> dict[str, object]: + if source == "extra_headers": + return {"extra_headers": {"anthropic-beta": malformed}} + return {"anthropic_beta": malformed} + + +@_MALFORMED_BETAS +@_MALFORMED_BETA_SOURCES +def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400_naming_its_field( + malformed: object, source: str, field: str +) -> None: with pytest.raises(AnthropicError) as excinfo: - VertexAIAnthropicConfig().transform_request( - model="claude-opus-4-6", - messages=[{"role": "user", "content": "Hello"}], - optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta}, - litellm_params={}, - headers={}, - ) + _vertex_request(_malformed_beta_params(source, malformed), {}) assert excinfo.value.status_code == 400 + assert excinfo.value.message.startswith(f"{field} must be") + + +@_MALFORMED_BETAS +@_MALFORMED_BETA_SOURCES +@pytest.mark.parametrize("auto_beta", [False, True], ids=["alone", "next_to_litellm_beta"]) +def test_vertex_ai_anthropic_malformed_anthropic_beta_is_dropped_under_drop_params( + local_beta_headers_config: None, malformed: object, source: str, field: str, auto_beta: bool +) -> None: + headers: Final[dict[str, str]] = {} + params: Final = {**_malformed_beta_params(source, malformed), **({"context_management": _CLEAR_TOOL_USES} if auto_beta else {})} + + result: Final = _vertex_request(params, headers, litellm_params={"drop_params": True}) + + assert result.get("anthropic_beta") == (["context-management-2025-06-27"] if auto_beta else None) + assert headers.get("anthropic-beta") == ("context-management-2025-06-27" if auto_beta else None) _VERTEX_MESSAGE_RESPONSE: Final = { @@ -1359,7 +1394,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = { } -def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config): +def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None: """A chat completion with `compaction` on a Vertex Claude deployment reaches Vertex with the beta in the anthropic-beta request header and never in the body field, the shape Vertex accepts (live, 2026-10-05).""" mock_response: Final = MagicMock() diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py new file mode 100644 index 00000000000..b23461823f8 --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py @@ -0,0 +1,34 @@ +from typing import Final + +import pytest + +from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_headers + +_HISTORY: Final = ({"role": "user", "content": "Hello"},) + + +@pytest.mark.parametrize( + "request_data", + [ + {"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": "sig"}]}]}, + {"messages": list(_HISTORY), "compaction": {"type": "summarize"}}, + ], + ids=["signed_compaction_block", "compaction_param"], +) +def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_data: dict[str, object]) -> None: + assert count_tokens_headers("token", request_data) == { + "Authorization": "Bearer token", + "anthropic-beta": "compact-2026-09-04", + } + + +@pytest.mark.parametrize( + "request_data", + [ + {"messages": list(_HISTORY)}, + {"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": ""}]}]}, + ], + ids=["plain", "unsigned_compaction_block"], +) +def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None: + assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"}