mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(vertex_ai): keep client body betas out of the Vertex body next to litellm betas, honor drop_params, and send the compaction beta on count_tokens
Litellm's own betas replace a client's body anthropic_beta again, as on main, so a body beta Vertex rejects no longer turns a 200 into a 400. A malformed beta drops under drop_params and names its field otherwise. inline-tools-2026-09-15 joins compact-2026-09-04 as header-only, since Vertex rejects both in the body. Vertex count_tokens sends compact-2026-09-04 when the request carries compaction.
This commit is contained in:
parent
c4862e8b49
commit
cdc280c867
6 changed files with 206 additions and 95 deletions
|
|
@ -22,7 +22,9 @@ if TYPE_CHECKING:
|
|||
|
||||
_NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
_VERTEX_HEADER_ONLY_BETAS: Final = frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
|
||||
_VERTEX_HEADER_ONLY_BETAS: Final = frozenset(
|
||||
{ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value, ANTHROPIC_BETA_HEADER_VALUES.INLINE_TOOLS_2026_09_15.value}
|
||||
)
|
||||
|
||||
|
||||
_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None)
|
||||
|
|
@ -30,18 +32,27 @@ _BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str
|
|||
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
|
||||
|
||||
|
||||
def _beta_names(value: object) -> frozenset[str]:
|
||||
def _beta_names(value: object) -> frozenset[str] | None:
|
||||
try:
|
||||
parsed: Final = _BETA_VALUE.validate_python(value)
|
||||
except ValidationError as error:
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"anthropic_beta must be a string or a list of strings, got {type(value).__name__}",
|
||||
) from error
|
||||
except ValidationError:
|
||||
return None
|
||||
names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or ()
|
||||
return frozenset(name.strip() for name in names if name.strip())
|
||||
|
||||
|
||||
def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]:
|
||||
names: Final = _beta_names(value)
|
||||
if names is not None:
|
||||
return names
|
||||
if drop_params:
|
||||
return frozenset()
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{field} must be a string or a list of strings, got {type(value).__name__}",
|
||||
)
|
||||
|
||||
|
||||
class VertexAIError(Exception):
|
||||
def __init__(self, status_code, message):
|
||||
self.status_code = status_code
|
||||
|
|
@ -162,12 +173,17 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"]))
|
||||
else frozenset[str]()
|
||||
)
|
||||
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {})
|
||||
data.pop("extra_headers", None)
|
||||
injected_betas: Final = frozenset(beta_set) | _beta_names(extra_headers.get("anthropic-beta"))
|
||||
client_body_betas: Final = _beta_names(_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"))
|
||||
injected_betas: Final = frozenset(beta_set) | _client_betas(
|
||||
extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params
|
||||
)
|
||||
client_body_betas: Final = _client_betas(
|
||||
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
|
||||
)
|
||||
|
||||
body_betas: Final = (injected_betas | client_body_betas) - _VERTEX_HEADER_ONLY_BETAS
|
||||
body_betas: Final = (injected_betas or client_body_betas) - _VERTEX_HEADER_ONLY_BETAS
|
||||
if body_betas:
|
||||
data["anthropic_beta"] = sorted(body_betas)
|
||||
elif "anthropic_beta" in data:
|
||||
|
|
@ -176,7 +192,11 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
injected_betas
|
||||
| compaction_betas
|
||||
| (client_body_betas & _VERTEX_HEADER_ONLY_BETAS)
|
||||
| _beta_names(_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"))
|
||||
| _client_betas(
|
||||
_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"),
|
||||
"The anthropic-beta header",
|
||||
drop_params,
|
||||
)
|
||||
)
|
||||
if header_betas:
|
||||
headers["anthropic-beta"] = ",".join(sorted(header_betas))
|
||||
|
|
|
|||
|
|
@ -6,11 +6,16 @@ Unlike Gemini models which use Google's token counting API, partner models use
|
|||
their respective publisher-specific count-tokens endpoints.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.anthropic.common_utils import requires_native_compaction_beta
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
_COUNT_TOKENS_PUBLISHERS: Final = (
|
||||
("claude", "anthropic"),
|
||||
|
|
@ -19,10 +24,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
|
|||
)
|
||||
|
||||
|
||||
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
|
||||
|
||||
|
||||
def count_tokens_publisher(model: str) -> str | None:
|
||||
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
|
||||
|
||||
|
||||
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
|
||||
authorization: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
if not requires_native_compaction_beta(
|
||||
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
|
||||
):
|
||||
return authorization
|
||||
return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}
|
||||
|
||||
|
||||
class VertexAIPartnerModelsTokenCounter(VertexBase):
|
||||
"""
|
||||
Token counter for Vertex AI Partner Models.
|
||||
|
|
@ -157,8 +174,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
api_base=litellm_params.get("api_base"),
|
||||
)
|
||||
|
||||
# Prepare headers
|
||||
headers: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
headers: Final = dict(count_tokens_headers(access_token, request_data))
|
||||
|
||||
# Get async HTTP client
|
||||
from litellm import LlmProviders
|
||||
|
|
|
|||
|
|
@ -794,6 +794,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
|
||||
COMPACT_2026_01_12 = "compact-2026-01-12"
|
||||
COMPACT_2026_09_04 = "compact-2026-09-04"
|
||||
INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15"
|
||||
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
|
||||
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
|
||||
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"
|
||||
|
|
|
|||
|
|
@ -132,7 +132,9 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out():
|
|||
assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "")
|
||||
|
||||
|
||||
def _validate_vertex_headers(client_headers, messages, optional_params=None):
|
||||
def _validate_vertex_headers(
|
||||
client_headers: dict[str, str], messages: list[object], optional_params: dict[str, object] | None = None
|
||||
) -> dict[str, str]:
|
||||
config = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
litellm_params = {
|
||||
"vertex_ai_project": "test-project",
|
||||
|
|
@ -829,7 +831,7 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f
|
|||
assert missing == []
|
||||
|
||||
|
||||
_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"}
|
||||
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -849,22 +851,25 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "
|
|||
ids=["compaction_param", "signed_compaction_block"],
|
||||
)
|
||||
def test_native_compaction_reaches_vertex_with_compact_2026_09_04_beta(
|
||||
local_beta_headers_config, client_headers, optional_params, messages
|
||||
):
|
||||
local_beta_headers_config: None,
|
||||
client_headers: dict[str, str],
|
||||
optional_params: dict[str, object],
|
||||
messages: list[object],
|
||||
) -> None:
|
||||
"""Vertex serves on-demand compaction only behind compact-2026-09-04 (Google Cloud: beta on
|
||||
https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05)
|
||||
and answers 400 "compaction: Extra inputs are not permitted" without it, so the beta rides
|
||||
along with the `compaction` param or a signed compaction block exactly once, whether or not
|
||||
the client sent it, and survives the Vertex beta filter."""
|
||||
headers = _validate_vertex_headers(client_headers, messages, optional_params)
|
||||
headers: Final = _validate_vertex_headers(client_headers, messages, optional_params)
|
||||
|
||||
filtered = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai")
|
||||
filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai")
|
||||
|
||||
assert filtered["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
|
||||
|
||||
|
||||
def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out():
|
||||
headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}])
|
||||
def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None:
|
||||
headers: Final = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}])
|
||||
|
||||
assert "compact-2026-09-04" not in headers.get("anthropic-beta", "")
|
||||
|
||||
|
|
@ -882,7 +887,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = {
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config):
|
||||
async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None:
|
||||
"""A /v1/messages compaction request to a Vertex Claude deployment reaches Vertex with the beta in the
|
||||
anthropic-beta request header and never in the body, the shape Vertex accepts (live, 2026-10-05)."""
|
||||
mock_response: Final = MagicMock()
|
||||
|
|
|
|||
|
|
@ -430,8 +430,10 @@ def test_vertex_ai_anthropic_extra_headers_beta_propagation():
|
|||
|
||||
def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(local_beta_headers_config: None) -> None:
|
||||
"""The chat path runs the Vertex beta filter over both the header and the `anthropic_beta` body field right
|
||||
before the request goes out, so inline-tools-2026-09-15 must survive both (source and date in
|
||||
tests/unit/test_anthropic_beta_headers_filtering.py)."""
|
||||
before the request goes out, so inline-tools-2026-09-15 must survive it as the header (source and date in
|
||||
tests/unit/test_anthropic_beta_headers_filtering.py) and stay out of the body field, which Vertex rejects
|
||||
with 400 "Unexpected value(s) `inline-tools-2026-09-15` for the `anthropic_beta` parameter" (live, global,
|
||||
claude-sonnet-5-5, 2026-10-20)."""
|
||||
config: Final = VertexAIAnthropicConfig()
|
||||
headers: Final[dict[str, str]] = {}
|
||||
optional_params: Final[dict[str, object]] = {
|
||||
|
|
@ -452,7 +454,7 @@ def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(loc
|
|||
)
|
||||
|
||||
assert filtered_headers["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1
|
||||
assert "inline-tools-2026-09-15" in filtered_request["anthropic_beta"]
|
||||
assert "inline-tools-2026-09-15" not in filtered_request.get("anthropic_beta", [])
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas():
|
||||
|
|
@ -1189,7 +1191,18 @@ def test_refusal_stop_details_kept_through_vertex_anthropic_transform():
|
|||
assert provider_fields["stop_details"] == ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"]
|
||||
|
||||
|
||||
_SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "signature": "sig"}
|
||||
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"}
|
||||
_CLEAR_TOOL_USES: Final = {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
|
||||
|
||||
def _vertex_request(optional_params: dict[str, object], headers: dict[str, str], litellm_params: dict[str, object] | None = None) -> dict:
|
||||
return VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={"max_tokens": 100, "is_vertex_request": True, **optional_params},
|
||||
litellm_params=litellm_params or {},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1214,18 +1227,17 @@ _SIGNED_COMPACTION_BLOCK = {"type": "compaction", "content": "summary so far", "
|
|||
ids=["compaction_param", "signed_compaction_block_sent_back"],
|
||||
)
|
||||
def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta(
|
||||
local_beta_headers_config, optional_params, messages
|
||||
):
|
||||
local_beta_headers_config: None, optional_params: dict[str, object], messages: list[dict[str, object]]
|
||||
) -> None:
|
||||
"""The chat path skips the Anthropic beta injection for Vertex requests, so the Vertex config
|
||||
adds compact-2026-09-04 itself (Google Cloud: beta on
|
||||
https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05)
|
||||
as the HTTP header only, since Vertex answered 400 "Unexpected value(s) `compact-2026-09-04`
|
||||
for the `anthropic_beta` parameter" with the name in the body field (us-east5,
|
||||
claude-sonnet-4-6, 2026-10-05), and the header must survive the Vertex beta filter."""
|
||||
config = VertexAIAnthropicConfig()
|
||||
headers = {}
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result = config.transform_request(
|
||||
result: Final = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
|
|
@ -1240,111 +1252,134 @@ def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta(
|
|||
assert "compact-2026-09-04" not in filtered_body.get("anthropic_beta", [])
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out():
|
||||
headers = {}
|
||||
def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={"max_tokens": 100, "is_vertex_request": True},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
result: Final = _vertex_request({}, headers)
|
||||
|
||||
assert "compact-2026-09-04" not in result.get("anthropic_beta", [])
|
||||
assert "compact-2026-09-04" not in headers.get("anthropic-beta", "")
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(local_beta_headers_config):
|
||||
headers = {}
|
||||
def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(
|
||||
local_beta_headers_config: None,
|
||||
) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={
|
||||
"max_tokens": 100,
|
||||
"is_vertex_request": True,
|
||||
result: Final = _vertex_request(
|
||||
{
|
||||
"compaction": {"type": "summarize"},
|
||||
"extra_headers": {"anthropic-beta": "compact-2026-09-04,interleaved-thinking-2025-05-14"},
|
||||
},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
headers,
|
||||
)
|
||||
|
||||
assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
|
||||
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(local_beta_headers_config):
|
||||
headers = {"anthropic-beta": "context-1m-2025-08-07"}
|
||||
def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(
|
||||
local_beta_headers_config: None,
|
||||
) -> None:
|
||||
headers: Final = {"anthropic-beta": "context-1m-2025-08-07"}
|
||||
|
||||
result = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
result: Final = _vertex_request({"compaction": {"type": "summarize"}}, headers)
|
||||
|
||||
assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"]
|
||||
assert "anthropic_beta" not in result
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_client_sent_body_compact_2026_09_04_moves_to_the_header(local_beta_headers_config):
|
||||
"""Vertex rejects `compact-2026-09-04` inside the body `anthropic_beta` field with 400 "Unexpected value(s)"
|
||||
(live, 2026-10-05), so a client that opts in through that field still gets the beta as a header and keeps
|
||||
its other betas in the body."""
|
||||
headers = {}
|
||||
@pytest.mark.parametrize("header_only_beta", ["compact-2026-09-04", "inline-tools-2026-09-15"])
|
||||
def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_header(
|
||||
local_beta_headers_config: None, header_only_beta: str
|
||||
) -> None:
|
||||
"""Vertex rejects these betas inside the body `anthropic_beta` field with 400 "Unexpected value(s)" and
|
||||
accepts them as the header (live, global, claude-sonnet-5-5, 2026-10-20), so a client that opts in through that field still
|
||||
gets the beta as a header and keeps its other betas in the body."""
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={
|
||||
"max_tokens": 100,
|
||||
"is_vertex_request": True,
|
||||
"compaction": {"type": "summarize"},
|
||||
"anthropic_beta": ["compact-2026-09-04", "interleaved-thinking-2025-05-14"],
|
||||
},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
result: Final = _vertex_request(
|
||||
{"anthropic_beta": [header_only_beta, "interleaved-thinking-2025-05-14"]}, headers
|
||||
)
|
||||
|
||||
assert headers["anthropic-beta"] == "compact-2026-09-04"
|
||||
assert headers["anthropic-beta"] == header_only_beta
|
||||
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"client_beta,header",
|
||||
[
|
||||
("context-1m-2025-08-07", "context-management-2025-06-27"),
|
||||
("inline-tools-2026-09-15", "context-management-2025-06-27,inline-tools-2026-09-15"),
|
||||
],
|
||||
ids=["body_beta", "header_only_beta"],
|
||||
)
|
||||
def test_vertex_ai_anthropic_litellm_betas_replace_the_client_body_betas(
|
||||
local_beta_headers_config: None, client_beta: str, header: str
|
||||
) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers)
|
||||
|
||||
assert result["anthropic_beta"] == ["context-management-2025-06-27"]
|
||||
assert headers["anthropic-beta"] == header
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"anthropic_beta",
|
||||
[["interleaved-thinking-2025-05-14"], ("interleaved-thinking-2025-05-14",)],
|
||||
ids=["list", "sdk_tuple"],
|
||||
)
|
||||
def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta):
|
||||
headers = {}
|
||||
def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta: object) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta},
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
result: Final = _vertex_request({"anthropic_beta": anthropic_beta}, headers)
|
||||
|
||||
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
|
||||
assert "anthropic-beta" not in headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize("anthropic_beta", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"])
|
||||
def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400(anthropic_beta):
|
||||
_MALFORMED_BETAS: Final = pytest.mark.parametrize(
|
||||
"malformed", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"]
|
||||
)
|
||||
_MALFORMED_BETA_SOURCES: Final = pytest.mark.parametrize(
|
||||
"source,field",
|
||||
[("anthropic_beta", "anthropic_beta"), ("extra_headers", "The anthropic-beta header")],
|
||||
ids=["body_field", "extra_header"],
|
||||
)
|
||||
|
||||
|
||||
def _malformed_beta_params(source: str, malformed: object) -> dict[str, object]:
|
||||
if source == "extra_headers":
|
||||
return {"extra_headers": {"anthropic-beta": malformed}}
|
||||
return {"anthropic_beta": malformed}
|
||||
|
||||
|
||||
@_MALFORMED_BETAS
|
||||
@_MALFORMED_BETA_SOURCES
|
||||
def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400_naming_its_field(
|
||||
malformed: object, source: str, field: str
|
||||
) -> None:
|
||||
with pytest.raises(AnthropicError) as excinfo:
|
||||
VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={"max_tokens": 100, "is_vertex_request": True, "anthropic_beta": anthropic_beta},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
_vertex_request(_malformed_beta_params(source, malformed), {})
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert excinfo.value.message.startswith(f"{field} must be")
|
||||
|
||||
|
||||
@_MALFORMED_BETAS
|
||||
@_MALFORMED_BETA_SOURCES
|
||||
@pytest.mark.parametrize("auto_beta", [False, True], ids=["alone", "next_to_litellm_beta"])
|
||||
def test_vertex_ai_anthropic_malformed_anthropic_beta_is_dropped_under_drop_params(
|
||||
local_beta_headers_config: None, malformed: object, source: str, field: str, auto_beta: bool
|
||||
) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
params: Final = {**_malformed_beta_params(source, malformed), **({"context_management": _CLEAR_TOOL_USES} if auto_beta else {})}
|
||||
|
||||
result: Final = _vertex_request(params, headers, litellm_params={"drop_params": True})
|
||||
|
||||
assert result.get("anthropic_beta") == (["context-management-2025-06-27"] if auto_beta else None)
|
||||
assert headers.get("anthropic-beta") == ("context-management-2025-06-27" if auto_beta else None)
|
||||
|
||||
|
||||
_VERTEX_MESSAGE_RESPONSE: Final = {
|
||||
|
|
@ -1359,7 +1394,7 @@ _VERTEX_MESSAGE_RESPONSE: Final = {
|
|||
}
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config):
|
||||
def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None:
|
||||
"""A chat completion with `compaction` on a Vertex Claude deployment reaches Vertex with the beta in the
|
||||
anthropic-beta request header and never in the body field, the shape Vertex accepts (live, 2026-10-05)."""
|
||||
mock_response: Final = MagicMock()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_headers
|
||||
|
||||
_HISTORY: Final = ({"role": "user", "content": "Hello"},)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data",
|
||||
[
|
||||
{"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": "sig"}]}]},
|
||||
{"messages": list(_HISTORY), "compaction": {"type": "summarize"}},
|
||||
],
|
||||
ids=["signed_compaction_block", "compaction_param"],
|
||||
)
|
||||
def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_data: dict[str, object]) -> None:
|
||||
assert count_tokens_headers("token", request_data) == {
|
||||
"Authorization": "Bearer token",
|
||||
"anthropic-beta": "compact-2026-09-04",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data",
|
||||
[
|
||||
{"messages": list(_HISTORY)},
|
||||
{"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": ""}]}]},
|
||||
],
|
||||
ids=["plain", "unsigned_compaction_block"],
|
||||
)
|
||||
def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None:
|
||||
assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"}
|
||||
Loading…
Add table
Reference in a new issue