fix(vertex_ai): keep client body betas out of the Vertex body next to litellm betas, honor drop_params, and send the compaction beta on count_tokens

Litellm's own betas replace a client's body anthropic_beta again, as on main, so a body beta
Vertex rejects no longer turns a 200 into a 400. A malformed beta drops under drop_params and
names its field otherwise. inline-tools-2026-09-15 joins compact-2026-09-04 as header-only, since
Vertex rejects both in the body. Vertex count_tokens sends compact-2026-09-04 when the request
carries compaction.
This commit is contained in:
mateo-berri 2026-10-20 04:46:31 -07:00
parent c4862e8b49
commit cdc280c867
6 changed files with 206 additions and 95 deletions

View file

@ -22,7 +22,9 @@ if TYPE_CHECKING:
_NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object])
_VERTEX_HEADER_ONLY_BETAS: Final = frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
_VERTEX_HEADER_ONLY_BETAS: Final = frozenset(
{ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value, ANTHROPIC_BETA_HEADER_VALUES.INLINE_TOOLS_2026_09_15.value}
)
_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None)
@ -30,18 +32,27 @@ _BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def _beta_names(value: object) -> frozenset[str]:
def _beta_names(value: object) -> frozenset[str] | None:
try:
parsed: Final = _BETA_VALUE.validate_python(value)
except ValidationError as error:
raise AnthropicError(
status_code=400,
message=f"anthropic_beta must be a string or a list of strings, got {type(value).__name__}",
) from error
except ValidationError:
return None
names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or ()
return frozenset(name.strip() for name in names if name.strip())
def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]:
names: Final = _beta_names(value)
if names is not None:
return names
if drop_params:
return frozenset()
raise AnthropicError(
status_code=400,
message=f"{field} must be a string or a list of strings, got {type(value).__name__}",
)
class VertexAIError(Exception):
def __init__(self, status_code, message):
self.status_code = status_code
@ -162,12 +173,17 @@ class VertexAIAnthropicConfig(AnthropicConfig):
if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"]))
else frozenset[str]()
)
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {})
data.pop("extra_headers", None)
injected_betas: Final = frozenset(beta_set) | _beta_names(extra_headers.get("anthropic-beta"))
client_body_betas: Final = _beta_names(_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"))
injected_betas: Final = frozenset(beta_set) | _client_betas(
extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params
)
client_body_betas: Final = _client_betas(
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
)
body_betas: Final = (injected_betas | client_body_betas) - _VERTEX_HEADER_ONLY_BETAS
body_betas: Final = (injected_betas or client_body_betas) - _VERTEX_HEADER_ONLY_BETAS
if body_betas:
data["anthropic_beta"] = sorted(body_betas)
elif "anthropic_beta" in data:
@ -176,7 +192,11 @@ class VertexAIAnthropicConfig(AnthropicConfig):
injected_betas
| compaction_betas
| (client_body_betas & _VERTEX_HEADER_ONLY_BETAS)
| _beta_names(_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"))
| _client_betas(
_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"),
"The anthropic-beta header",
drop_params,
)
)
if header_betas:
headers["anthropic-beta"] = ",".join(sorted(header_betas))

View file

@ -6,11 +6,16 @@ Unlike Gemini models which use Google's token counting API, partner models use
their respective publisher-specific count-tokens endpoints.
"""
from collections.abc import Mapping, Sequence
from typing import Any, Final
from pydantic import TypeAdapter
from litellm.llms.anthropic.common_utils import requires_native_compaction_beta
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
_COUNT_TOKENS_PUBLISHERS: Final = (
("claude", "anthropic"),
@ -19,10 +24,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
)
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def count_tokens_publisher(model: str) -> str | None:
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
authorization: Final = {"Authorization": f"Bearer {access_token}"}
if not requires_native_compaction_beta(
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
):
return authorization
return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}
class VertexAIPartnerModelsTokenCounter(VertexBase):
"""
Token counter for Vertex AI Partner Models.
@ -157,8 +174,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
api_base=litellm_params.get("api_base"),
)
# Prepare headers
headers: Final = {"Authorization": f"Bearer {access_token}"}
headers: Final = dict(count_tokens_headers(access_token, request_data))
# Get async HTTP client
from litellm import LlmProviders

View file

@ -794,6 +794,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
COMPACT_2026_01_12 = "compact-2026-01-12"
COMPACT_2026_09_04 = "compact-2026-09-04"
INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15"
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"

View file

@ -132,7 +132,9 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out():
assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "")
def _validate_vertex_headers(client_headers, messages, optional_params=None):
def _validate_vertex_headers(
client_headers: dict[str, str], messages: list[object], optional_params: dict[str, object] | None = None
) -> dict[str, str]:
config = VertexAIPartnerModelsAnthropicMessagesConfig()
litellm_params = {
"vertex_ai_project": "test-project",
@ -829,7 +831,7 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f
assert missing == []
_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"}
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"}
@pytest.mark.parametrize(
@ -849,22 +851,25 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "
ids=["compaction_param", "signed_compaction_block"],
)
def test_native_compaction_reaches_vertex_with_compact_2026_09_04_beta(
local_beta_headers_config, client_headers, optional_params, messages
):
local_beta_headers_config: None,
client_headers: dict[str, str],
optional_params: dict[str, object],
messages: list[object],
) -> None:
"""Vertex serves on-demand compaction only behind compact-2026-09-04 (Google Cloud: beta on
https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05)
and answers 400 "compaction: Extra inputs are not permitted" without it, so the beta rides
along with the `compaction` param or a signed compaction block exactly once, whether or not
the client sent it, and survives the Vertex beta filter."""
headers = _validate_vertex_headers(client_headers, messages, optional_params)
headers: Final = _validate_vertex_headers(client_headers, messages, optional_params)
filtered = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai")
filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai")
assert filtered["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out():
headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}])
def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None:
headers: Final = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}])
assert "compact-2026-09-04" not in headers.get("anthropic-beta", "")
@ -882,7 +887,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = {
@pytest.mark.asyncio
async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config):
async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None:
"""A /v1/messages compaction request to a Vertex Claude deployment reaches Vertex with the beta in the
anthropic-beta request header and never in the body, the shape Vertex accepts (live, 2026-10-05)."""
mock_response: Final = MagicMock()

View file

@ -430,8 +430,10 @@ def test_vertex_ai_anthropic_extra_headers_beta_propagation():
def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(local_beta_headers_config: None) -> None:
"""The chat path runs the Vertex beta filter over both the header and the `anthropic_beta` body field right
before the request goes out, so inline-tools-2026-09-15 must survive both (source and date in
tests/unit/test_anthropic_beta_headers_filtering.py)."""
before the request goes out, so inline-tools-2026-09-15 must survive it as the header (source and date in
tests/unit/test_anthropic_beta_headers_filtering.py) and stay out of the body field, which Vertex rejects
with 400 "Unexpected value(s) `inline-tools-2026-09-15` for the `anthropic_beta` parameter" (live, global,
claude-sonnet-5-5, 2026-10-20)."""
config: Final = VertexAIAnthropicConfig()
headers: Final[dict[str, str]] = {}
optional_params: Final[dict[str, object]] = {
@ -452,7 +454,7 @@ def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(loc
)
assert filtered_headers["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1
assert "inline-tools-2026-09-15" in filtered_request["anthropic_beta"]
assert "inline-tools-2026-09-15" not in filtered_request.get("anthropic_beta", [])
def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas():
@ -1189,7 +1191,18 @@ def test_refusal_stop_details_kept_through_vertex_anthropic_transform():
assert provider_fields["stop_details"] == ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"]
_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"}
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"}
_CLEAR_TOOL_USES: Final = {"edits": [{"type": "clear_tool_uses_20250919"}]}
def _vertex_request(optional_params: dict[str, object], headers: dict[str, str], litellm_params: dict[str, object] | None = None) -> dict:
return VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Hello"}],
optional_params={"max_tokens": 100, "is_vertex_request": True, **optional_params},
litellm_params=litellm_params or {},
headers=headers,
)
@pytest.mark.parametrize(
@ -1214,18 +1227,17 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "
ids=["compaction_param", "signed_compaction_block_sent_back"],
)
def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta(
local_beta_headers_config, optional_params, messages
):
local_beta_headers_config: None, optional_params: dict[str, object], messages: list[dict[str, object]]
) -> None:
"""The chat path skips the Anthropic beta injection for Vertex requests, so the Vertex config
adds compact-2026-09-04 itself (Google Cloud: beta on
https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05)
as the HTTP header only, since Vertex answered 400 "Unexpected value(s) `compact-2026-09-04`
for the `anthropic_beta` parameter" with the name in the body field (us-east5,
claude-sonnet-4-6, 2026-10-05), and the header must survive the Vertex beta filter."""
config = VertexAIAnthropicConfig()
headers = {}
headers: Final[dict[str, str]] = {}
result = config.transform_request(
result: Final = VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=messages,
optional_params=optional_params,
@ -1240,111 +1252,134 @@ def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta(
assert "compact-2026-09-04" not in filtered_body.get("anthropic_beta", [])
def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out():
headers = {}
def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None:
headers: Final[dict[str, str]] = {}
result = VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Hello"}],
optional_params={"max_tokens": 100, "is_vertex_request": True},
litellm_params={},
headers=headers,
)
result: Final = _vertex_request({}, headers)
assert "compact-2026-09-04" not in result.get("anthropic_beta", [])
assert "compact-2026-09-04" not in headers.get("anthropic-beta", "")
def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(local_beta_headers_config):
headers = {}
def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(
local_beta_headers_config: None,
) -> None:
headers: Final[dict[str, str]] = {}
result = VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Hello"}],
optional_params={
"max_tokens": 100,
"is_vertex_request": True,
result: Final = _vertex_request(
{
"compaction": {"type": "summarize"},
"extra_headers": {"anthropic-beta": "compact-2026-09-04,interleaved-thinking-2025-05-14"},
},
litellm_params={},
headers=headers,
headers,
)
assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(local_beta_headers_config):
headers = {"anthropic-beta": "context-1m-2025-08-07"}
def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(
local_beta_headers_config: None,
) -> None:
headers: Final = {"anthropic-beta": "context-1m-2025-08-07"}
result = VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Hello"}],
optional_params={"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}},
litellm_params={},
headers=headers,
)
result: Final = _vertex_request({"compaction": {"type": "summarize"}}, headers)
assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"]
assert "anthropic_beta" not in result
def test_vertex_ai_anthropic_client_sent_body_compact_2026_09_04_moves_to_the_header(local_beta_headers_config):
"""Vertex rejects `compact-2026-09-04` inside the body `anthropic_beta` field with 400 "Unexpected value(s)"
(live, 2026-10-05), so a client that opts in through that field still gets the beta as a header and keeps
its other betas in the body."""
headers = {}
@pytest.mark.parametrize("header_only_beta", ["compact-2026-09-04", "inline-tools-2026-09-15"])
def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_header(
local_beta_headers_config: None, header_only_beta: str
) -> None:
"""Vertex rejects these betas inside the body `anthropic_beta` field with 400 "Unexpected value(s)" and
accepts them as the header (live, global, claude-sonnet-5-5, 2026-10-20), so a client that opts in through that field still
gets the beta as a header and keeps its other betas in the body."""
headers: Final[dict[str, str]] = {}
result = VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Hello"}],
optional_params={
"max_tokens": 100,
"is_vertex_request": True,
"compaction": {"type": "summarize"},
"anthropic_beta": ["compact-2026-09-04", "interleaved-thinking-2025-05-14"],
},
litellm_params={},
headers=headers,
result: Final = _vertex_request(
{"anthropic_beta": [header_only_beta, "interleaved-thinking-2025-05-14"]}, headers
)
assert headers["anthropic-beta"] == "compact-2026-09-04"
assert headers["anthropic-beta"] == header_only_beta
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
@pytest.mark.parametrize(
"client_beta,header",
[
("context-1m-2025-08-07", "context-management-2025-06-27"),
("inline-tools-2026-09-15", "context-management-2025-06-27,inline-tools-2026-09-15"),
],
ids=["body_beta", "header_only_beta"],
)
def test_vertex_ai_anthropic_litellm_betas_replace_the_client_body_betas(
local_beta_headers_config: None, client_beta: str, header: str
) -> None:
headers: Final[dict[str, str]] = {}
result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers)
assert result["anthropic_beta"] == ["context-management-2025-06-27"]
assert headers["anthropic-beta"] == header
@pytest.mark.parametrize(
"anthropic_beta",
[["interleaved-thinking-2025-05-14"], ("interleaved-thinking-2025-05-14",)],
ids=["list", "sdk_tuple"],
)
def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta):
headers = {}
def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta: object) -> None:
headers: Final[dict[str, str]] = {}
result = VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Hello"}],
optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta},
litellm_params={},
headers=headers,
)
result: Final = _vertex_request({"anthropic_beta": anthropic_beta}, headers)
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
assert "anthropic-beta" not in headers
@pytest.mark.parametrize("anthropic_beta", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"])
def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400(anthropic_beta):
_MALFORMED_BETAS: Final = pytest.mark.parametrize(
"malformed", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"]
)
_MALFORMED_BETA_SOURCES: Final = pytest.mark.parametrize(
"source,field",
[("anthropic_beta", "anthropic_beta"), ("extra_headers", "The anthropic-beta header")],
ids=["body_field", "extra_header"],
)
def _malformed_beta_params(source: str, malformed: object) -> dict[str, object]:
if source == "extra_headers":
return {"extra_headers": {"anthropic-beta": malformed}}
return {"anthropic_beta": malformed}
@_MALFORMED_BETAS
@_MALFORMED_BETA_SOURCES
def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400_naming_its_field(
malformed: object, source: str, field: str
) -> None:
with pytest.raises(AnthropicError) as excinfo:
VertexAIAnthropicConfig().transform_request(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Hello"}],
optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta},
litellm_params={},
headers={},
)
_vertex_request(_malformed_beta_params(source, malformed), {})
assert excinfo.value.status_code == 400
assert excinfo.value.message.startswith(f"{field} must be")
@_MALFORMED_BETAS
@_MALFORMED_BETA_SOURCES
@pytest.mark.parametrize("auto_beta", [False, True], ids=["alone", "next_to_litellm_beta"])
def test_vertex_ai_anthropic_malformed_anthropic_beta_is_dropped_under_drop_params(
local_beta_headers_config: None, malformed: object, source: str, field: str, auto_beta: bool
) -> None:
headers: Final[dict[str, str]] = {}
params: Final = {**_malformed_beta_params(source, malformed), **({"context_management": _CLEAR_TOOL_USES} if auto_beta else {})}
result: Final = _vertex_request(params, headers, litellm_params={"drop_params": True})
assert result.get("anthropic_beta") == (["context-management-2025-06-27"] if auto_beta else None)
assert headers.get("anthropic-beta") == ("context-management-2025-06-27" if auto_beta else None)
_VERTEX_MESSAGE_RESPONSE: Final = {
@ -1359,7 +1394,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = {
}
def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config):
def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None:
"""A chat completion with `compaction` on a Vertex Claude deployment reaches Vertex with the beta in the
anthropic-beta request header and never in the body field, the shape Vertex accepts (live, 2026-10-05)."""
mock_response: Final = MagicMock()

View file

@ -0,0 +1,34 @@
from typing import Final
import pytest
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_headers
_HISTORY: Final = ({"role": "user", "content": "Hello"},)
@pytest.mark.parametrize(
"request_data",
[
{"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": "sig"}]}]},
{"messages": list(_HISTORY), "compaction": {"type": "summarize"}},
],
ids=["signed_compaction_block", "compaction_param"],
)
def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_data: dict[str, object]) -> None:
assert count_tokens_headers("token", request_data) == {
"Authorization": "Bearer token",
"anthropic-beta": "compact-2026-09-04",
}
@pytest.mark.parametrize(
"request_data",
[
{"messages": list(_HISTORY)},
{"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": ""}]}]},
],
ids=["plain", "unsigned_compaction_block"],
)
def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None:
assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"}