diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
index ed61707f99e..3084b0fde07 100644
--- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
+++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
@@ -20,6 +20,7 @@ from litellm.types.utils import LiteLLMPydanticObjectBase, LlmProviders, TokenCo
RUNTIME_TOKENIZER_TYPE: Final = "bedrock_api"
MANTLE_TOKENIZER_TYPE: Final = "bedrock_mantle_api"
+_RUNTIME_FALLBACK_STATUSES: Final = frozenset({400, 404})
class _CountTokensReply(LiteLLMPydanticObjectBase):
@@ -46,14 +47,14 @@ CountTokensOutcome = CountedTokens | CountTokensFailure
def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: str) -> bool:
return (
isinstance(outcome, CountTokensFailure)
- and outcome.status_code == 400
+ and outcome.status_code in _RUNTIME_FALLBACK_STATUSES
and is_mantle_claude_model(resolved_model)
)
class BedrockTokenCounter(BaseTokenCounter):
"""Token counter for AWS Bedrock: bedrock-runtime CountTokens first, and Anthropic's count_tokens
- on bedrock-mantle for the Claude models bedrock-runtime answers 400 for"""
+ on bedrock-mantle for Claude models bedrock-runtime answers 400 or 404 for"""
def __init__(
self,
diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py
index 92b3b85dc51..b18c322b3a4 100644
--- a/litellm/llms/gemini/common_utils.py
+++ b/litellm/llms/gemini/common_utils.py
@@ -499,12 +499,23 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
import copy
from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
+ from litellm.llms.vertex_ai.count_tokens.context import (
+ VertexGeminiCountContextError,
+ count_context_error_response,
+ vertex_gemini_count_context,
+ )
+
+ count_context: Final = vertex_gemini_count_context(system=system, tools=tools)
+ if isinstance(count_context, VertexGeminiCountContextError):
+ return count_context_error_response(count_context, request_model, model_to_use)
deployment = deployment or {}
count_tokens_params_request: Final = copy.deepcopy(deployment.get("litellm_params", {}))
count_tokens_params: Final = {
"model": model_to_use,
"contents": contents,
+ "system_instruction": count_context.system_instruction,
+ "tools": count_context.tools,
}
count_tokens_params_request.update(count_tokens_params)
result: Final = await GoogleAIStudioTokenCounter().acount_tokens(
diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py
index ca47ac67957..07e05b2b295 100644
--- a/litellm/llms/gemini/count_tokens/handler.py
+++ b/litellm/llms/gemini/count_tokens/handler.py
@@ -6,7 +6,7 @@ from pydantic import ConfigDict, RootModel, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
-from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
@@ -41,12 +41,12 @@ class GoogleAIStudioTokenCounter:
"""
import copy
- from google.genai.types import FunctionResponse
-
# Handle None or empty contents
if not contents:
return contents
+ from google.genai.types import FunctionResponse
+
cleaned_contents: Final = copy.deepcopy(contents)
for content in cleaned_contents:
@@ -67,6 +67,28 @@ class GoogleAIStudioTokenCounter:
base_url: Final = api_base or "https://generativelanguage.googleapis.com"
return f"{base_url}/v1beta/models/{model}:countTokens"
+ def __init__(self, client: AsyncHTTPHandler | None = None) -> None:
+ super().__init__()
+ self._client: Final = client
+
+ def _build_count_tokens_request_body(
+ self,
+ model: str,
+ contents: object,
+ system_instruction: Mapping[str, object] | None,
+ tools: Sequence[Mapping[str, object]] | None,
+ ) -> Mapping[str, object]:
+ if system_instruction is None and not tools:
+ return {"contents": contents}
+
+ generate_content_request: Final = {
+ "model": model if model.startswith("models/") else f"models/{model}",
+ "contents": contents,
+ **({"systemInstruction": dict(system_instruction)} if system_instruction is not None else {}),
+ **({"tools": [dict(tool) for tool in tools]} if tools else {}),
+ }
+ return {"generateContentRequest": generate_content_request}
+
async def validate_environment(
self,
api_base: str | None = None,
@@ -97,6 +119,8 @@ class GoogleAIStudioTokenCounter:
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
+ system_instruction: Mapping[str, object] | None = None,
+ tools: Sequence[Mapping[str, object]] | None = None,
**kwargs: object,
) -> dict[str, Any]:
"""
@@ -143,14 +167,19 @@ class GoogleAIStudioTokenCounter:
# Prepare request body - clean up contents to remove unsupported fields
cleaned_contents: Final = self._clean_contents_for_gemini_api(contents)
- request_body: Final = {"contents": cleaned_contents}
+ request_body: Final = self._build_count_tokens_request_body(
+ model=model,
+ contents=cleaned_contents,
+ system_instruction=system_instruction,
+ tools=tools,
+ )
- async_httpx_client: Final = get_async_httpx_client(
- llm_provider=LlmProviders.GEMINI,
+ async_httpx_client: Final = (
+ self._client if self._client is not None else get_async_httpx_client(llm_provider=LlmProviders.GEMINI)
)
try:
- response: Final = await async_httpx_client.post(url=url, headers=headers, json=request_body)
+ response: Final = await async_httpx_client.post(url=url, headers=headers, json=dict(request_body))
# Check for HTTP errors
response.raise_for_status()
diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py
index 45d8ab0b4c4..75684b06d05 100644
--- a/litellm/llms/openai/responses/count_tokens/transformation.py
+++ b/litellm/llms/openai/responses/count_tokens/transformation.py
@@ -157,9 +157,10 @@ class OpenAICountTokensConfig:
tools: list[dict[str, Any]],
) -> list[dict[str, object]]:
"""
- Transform OpenAI chat tools format to Responses API tools format.
+ Transform OpenAI chat and Anthropic tools to Responses API tools format.
Chat format: {"type": "function", "function": {"name": "...", "parameters": {...}}}
+ Anthropic format: {"name": "...", "input_schema": {...}}
Responses format: {"type": "function", "name": "...", "parameters": {...}}
"""
transformed: Final[list[dict[str, object]]] = []
@@ -175,6 +176,16 @@ class OpenAICountTokensConfig:
if "strict" in func:
item["strict"] = func["strict"]
transformed.append(item)
+ elif "input_schema" in tool and "name" in tool and tool.get("type") in (None, "custom"):
+ transformed.append(
+ {
+ "type": "function",
+ "name": tool["name"],
+ "description": tool.get("description", ""),
+ "parameters": tool["input_schema"],
+ "strict": bool(tool.get("strict")),
+ }
+ )
else:
# Pass through non-function tools (e.g., web_search, file_search)
transformed.append(tool)
diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py
index 7c96b190480..588416ff282 100644
--- a/litellm/llms/vertex_ai/common_utils.py
+++ b/litellm/llms/vertex_ai/common_utils.py
@@ -1343,30 +1343,56 @@ class VertexAITokenCounter(BaseTokenCounter):
original_response=result,
)
else:
+ from litellm.llms.vertex_ai.count_tokens.context import (
+ VertexGeminiCountContextError,
+ count_context_error_response,
+ vertex_gemini_count_context,
+ )
from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
from litellm.llms.vertex_ai.gemini.transformation import (
gemini_convert_messages_with_history, # pyright: ignore[reportPrivateUsage] # shared helper already used by gemini/chat, context_caching, and vertex_and_google_ai_studio_gemini
)
+ count_context: Final = vertex_gemini_count_context(system=system, tools=tools)
+ if isinstance(count_context, VertexGeminiCountContextError):
+ return count_context_error_response(count_context, request_model, model_to_use)
+
resolved_contents: Final = (
- contents if contents is not None else gemini_convert_messages_with_history(messages=messages or [])
+ contents
+ if contents is not None
+ else gemini_convert_messages_with_history(
+ messages=messages or [] # pyright: ignore[reportArgumentType] # preserve direct caller message handling
+ )
)
count_tokens_params: Final = {
"model": model_to_use,
"contents": resolved_contents,
+ "system_instruction": count_context.system_instruction,
+ "tools": count_context.tools,
}
count_tokens_params_request.update(count_tokens_params)
- result = await VertexAITokenCounter().acount_tokens(
- **count_tokens_params_request,
- )
+ try:
+ result = await VertexAITokenCounter().acount_tokens(
+ **count_tokens_params_request,
+ )
+ except litellm.APIError as error:
+ return TokenCountResponse(
+ total_tokens=0,
+ request_model=request_model,
+ model_used=model_to_use,
+ tokenizer_type="gemini",
+ error=True,
+ error_message=error.message,
+ status_code=error.status_code,
+ )
- if result is not None and "totalTokens" in result:
+ if "totalTokens" in result:
return TokenCountResponse(
total_tokens=result["totalTokens"],
request_model=request_model,
model_used=model_to_use,
- tokenizer_type=result.get("tokenizer_used", ""),
+ tokenizer_type=result.get("tokenizer_used", "gemini"),
original_response=result,
)
diff --git a/litellm/llms/vertex_ai/count_tokens/context.py b/litellm/llms/vertex_ai/count_tokens/context.py
new file mode 100644
index 00000000000..cb0bedf0238
--- /dev/null
+++ b/litellm/llms/vertex_ai/count_tokens/context.py
@@ -0,0 +1,144 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm.types.llms.anthropic import AnthropicMessagesTextParam
+from litellm.types.utils import TokenCountResponse
+
+
+@dataclass(frozen=True, slots=True)
+class VertexGeminiCountContext:
+ system_instruction: Mapping[str, object] | None
+ tools: tuple[Mapping[str, object], ...] | None
+
+
+@dataclass(frozen=True, slots=True)
+class VertexGeminiCountContextError:
+ error_message: str
+
+
+_SYSTEM_BLOCKS: Final = TypeAdapter(tuple[AnthropicMessagesTextParam, ...])
+_TOOLS: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
+_VERTEX_TOOL_KEYS: Final = MappingProxyType(
+ {
+ "function_declarations": "functionDeclarations",
+ "url_context": "urlContext",
+ "code_execution": "codeExecution",
+ }
+)
+
+
+def _system_parts(
+ system: object | None,
+) -> tuple[Mapping[str, JsonValue], ...] | VertexGeminiCountContextError:
+ if system is None:
+ return ()
+ if isinstance(system, str):
+ return ({"text": system},) if system else ()
+ try:
+ blocks: Final = _SYSTEM_BLOCKS.validate_python(system)
+ except ValidationError:
+ return VertexGeminiCountContextError(error_message="Gemini system instructions only support text blocks.")
+ return tuple({"text": block["text"]} for block in blocks)
+
+
+_INVALID_TOOLS: Final = VertexGeminiCountContextError(
+ error_message="tools must be a list of objects, and each function tool needs a JSON object schema."
+)
+
+
+def _has_object_schema(tool: Mapping[str, JsonValue]) -> bool:
+ function: Final = tool.get("function")
+ schema: Final = (
+ function.get("parameters") if isinstance(function, dict) else tool.get("input_schema", tool.get("parameters"))
+ )
+ return schema is None or isinstance(schema, dict)
+
+
+def _is_function_tool(tool: Mapping[str, JsonValue]) -> bool:
+ return isinstance(tool.get("name"), str) or "input_schema" in tool or "function" in tool
+
+
+def _is_responses_function_tool(tool: Mapping[str, JsonValue]) -> bool:
+ return tool.get("type") == "function" and "function" not in tool and "input_schema" not in tool
+
+
+def _chat_tool(tool: Mapping[str, JsonValue]) -> tuple[Mapping[str, object], ...]:
+ from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
+ from litellm.responses.litellm_completion_transformation.transformation import (
+ LiteLLMCompletionResponsesConfig,
+ )
+
+ if _is_responses_function_tool(tool):
+ chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
+ [dict(tool)] # pyright: ignore[reportArgumentType] # validated JSON dict in the Responses function tool shape
+ )
+ return tuple(chat_tools)
+ anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
+ [dict(tool)] # pyright: ignore[reportArgumentType] # validated JSON dict in an Anthropic or chat tool shape
+ )
+ return tuple(anthropic_tools)
+
+
+def _map_function_tools(
+ tools: object | None,
+) -> tuple[Mapping[str, object], ...] | None | VertexGeminiCountContextError:
+ try:
+ tool_values: Final = _TOOLS.validate_python(tools or ())
+ except ValidationError:
+ return _INVALID_TOOLS
+ function_tools: Final = tuple(tool for tool in tool_values if _is_function_tool(tool))
+ if not function_tools:
+ return None
+ if not all(_has_object_schema(tool) for tool in function_tools):
+ return _INVALID_TOOLS
+
+ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
+
+ chat_tools: Final = tuple(
+ chat_tool for tool in function_tools for chat_tool in _chat_tool(tool)
+ ) # comprehension-ok: flattens each tool's translation
+ try:
+ mapped_tools: Final = VertexGeminiConfig().map_function(
+ value=[dict(tool) for tool in chat_tools],
+ optional_params={},
+ )
+ except (AttributeError, TypeError, ValueError):
+ return _INVALID_TOOLS
+ return tuple(
+ MappingProxyType({_VERTEX_TOOL_KEYS.get(key, key): value for key, value in tool.items()})
+ for tool in mapped_tools
+ )
+
+
+def vertex_gemini_count_context(
+ system: object | None,
+ tools: object | None,
+) -> VertexGeminiCountContext | VertexGeminiCountContextError:
+ system_parts: Final = _system_parts(system)
+ if isinstance(system_parts, VertexGeminiCountContextError):
+ return system_parts
+ function_tools: Final = _map_function_tools(tools)
+ if isinstance(function_tools, VertexGeminiCountContextError):
+ return function_tools
+ return VertexGeminiCountContext(
+ system_instruction=MappingProxyType({"parts": system_parts}) if system_parts else None,
+ tools=function_tools,
+ )
+
+
+def count_context_error_response(
+ error: VertexGeminiCountContextError, request_model: str, model_used: str
+) -> TokenCountResponse:
+ return TokenCountResponse(
+ total_tokens=0,
+ request_model=request_model,
+ model_used=model_used,
+ tokenizer_type="gemini",
+ error=True,
+ error_message=error.error_message,
+ status_code=400,
+ )
diff --git a/litellm/llms/vertex_ai/count_tokens/handler.py b/litellm/llms/vertex_ai/count_tokens/handler.py
index 47a08ff054d..03970099f2e 100644
--- a/litellm/llms/vertex_ai/count_tokens/handler.py
+++ b/litellm/llms/vertex_ai/count_tokens/handler.py
@@ -1,3 +1,4 @@
+from collections.abc import Mapping, Sequence
from typing import Any, Final
from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
@@ -5,6 +6,19 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
class VertexAITokenCounter(GoogleAIStudioTokenCounter, VertexBase):
+ def _build_count_tokens_request_body(
+ self,
+ model: str,
+ contents: object,
+ system_instruction: Mapping[str, object] | None,
+ tools: Sequence[Mapping[str, object]] | None,
+ ) -> Mapping[str, object]:
+ return {
+ "contents": contents,
+ **({"systemInstruction": dict(system_instruction)} if system_instruction else {}),
+ **({"tools": [dict(tool) for tool in tools]} if tools else {}),
+ }
+
async def validate_environment(
self,
api_base: str | None = None,
@@ -16,29 +30,29 @@ class VertexAITokenCounter(GoogleAIStudioTokenCounter, VertexBase):
"""
Returns a Tuple of headers and url for the Vertex AI countTokens endpoint.
"""
- litellm_params = litellm_params or {}
- vertex_credentials: Final = self.get_vertex_ai_credentials(litellm_params=litellm_params)
- vertex_project = self.get_vertex_ai_project(litellm_params=litellm_params)
- vertex_location: Final = self.get_vertex_ai_location(litellm_params=litellm_params)
- _auth_header, vertex_project = await self._ensure_access_token_async(
+ resolved_litellm_params: Final = litellm_params or {}
+ vertex_credentials: Final = self.get_vertex_ai_credentials(litellm_params=resolved_litellm_params)
+ vertex_project: Final = self.get_vertex_ai_project(litellm_params=resolved_litellm_params)
+ vertex_location: Final = self.get_vertex_ai_location(litellm_params=resolved_litellm_params)
+ _auth_header, resolved_vertex_project = await self._ensure_access_token_async(
credentials=vertex_credentials,
project_id=vertex_project,
custom_llm_provider="vertex_ai",
)
- auth_header, api_base = self._get_token_and_url(
+ auth_header, url = self._get_token_and_url(
model=model,
gemini_api_key=None,
auth_header=_auth_header,
- vertex_project=vertex_project,
+ vertex_project=resolved_vertex_project,
vertex_location=vertex_location,
vertex_credentials=vertex_credentials,
stream=False,
custom_llm_provider="vertex_ai",
- api_base=None,
+ api_base=api_base,
mode="count_tokens",
)
headers = {
"Authorization": f"Bearer {auth_header}",
}
- return headers, api_base
+ return headers, url
diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py
index 8638bc57615..eabc360fd6d 100644
--- a/litellm/proxy/google_endpoints/endpoints.py
+++ b/litellm/proxy/google_endpoints/endpoints.py
@@ -1,9 +1,14 @@
+from collections.abc import Mapping
+from itertools import chain
from typing import Final
-from fastapi import APIRouter, Depends, Request, Response
+from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi.responses import ORJSONResponse
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
+from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types
from litellm.proxy._types import *
+from litellm.proxy._types import TokenCountRequest
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
@@ -12,11 +17,83 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
)
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
+
+class _GoogleCountTokensTextPart(BaseModel):
+ model_config = ConfigDict(frozen=True)
+ text: str
+
+
+class _GoogleCountTokensSystemInstruction(BaseModel):
+ model_config = ConfigDict(frozen=True)
+ parts: tuple[_GoogleCountTokensTextPart, ...]
+
+
+class _GoogleCountTokensFunctionDeclaration(BaseModel):
+ model_config = ConfigDict(frozen=True, populate_by_name=True)
+ name: str
+ description: str | None = None
+ parameters: JsonValue | None = None
+ parameters_json_schema: JsonValue | None = Field(default=None, alias="parametersJsonSchema")
+
+
+class _GoogleCountTokensTool(BaseModel):
+ model_config = ConfigDict(frozen=True, populate_by_name=True)
+ function_declarations: tuple[_GoogleCountTokensFunctionDeclaration, ...] = Field(
+ default=(), alias="functionDeclarations"
+ )
+
+
+class _GoogleCountTokensRequest(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True)
+ contents: tuple[Mapping[str, JsonValue], ...] = ()
+ system_instruction: _GoogleCountTokensSystemInstruction | None = Field(default=None, alias="systemInstruction")
+ tools: tuple[_GoogleCountTokensTool, ...] = ()
+
+
router: Final = APIRouter(
tags=["google genai endpoints"],
)
+def _input_schema(declaration: _GoogleCountTokensFunctionDeclaration) -> object:
+ if declaration.parameters_json_schema is not None:
+ return declaration.parameters_json_schema
+ if declaration.parameters is not None:
+ return normalize_json_schema_types(declaration.parameters)
+ return {"type": "object", "properties": {}}
+
+
+def _google_count_tokens_request(model_name: str, data: object) -> TokenCountRequest:
+ from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
+
+ validated_data: Final = TypeAdapter(_GoogleCountTokensRequest).validate_python(data)
+ contents: Final = [dict(content) for content in validated_data.contents]
+ messages: Final = (
+ GoogleGenAIAdapter().translate_generate_content_to_completion(model_name, contents).get("messages", [])
+ )
+ system: Final = (
+ [{"type": "text", "text": part.text} for part in validated_data.system_instruction.parts]
+ if validated_data.system_instruction is not None
+ else None
+ )
+ declarations: Final = chain.from_iterable(tool.function_declarations for tool in validated_data.tools)
+ tools: Final = [
+ {
+ "name": declaration.name,
+ **({"description": declaration.description} if declaration.description is not None else {}),
+ "input_schema": _input_schema(declaration),
+ }
+ for declaration in declarations
+ ]
+ return TokenCountRequest(
+ model=model_name,
+ contents=contents,
+ messages=messages,
+ system=system,
+ tools=tools or None,
+ )
+
+
@router.post(
"/v1beta/models/{model_name:path}:generateContent",
dependencies=[Depends(user_api_key_auth)],
@@ -168,23 +245,14 @@ async def google_count_tokens(request: Request, model_name: str):
}
```
"""
- from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
from litellm.proxy.common_utils.http_parsing_utils import read_request_body
from litellm.proxy.proxy_server import token_counter as internal_token_counter
data: Final = await read_request_body(request=request)
- contents: Final = data.get("contents", [])
- # Create TokenCountRequest for the internal endpoint
- from litellm.proxy._types import TokenCountRequest
-
- # Translate contents to openai format messages using the adapter
- messages = GoogleGenAIAdapter().translate_generate_content_to_completion(model_name, contents).get("messages", [])
-
- token_request: Final = TokenCountRequest(
- model=model_name,
- contents=contents,
- messages=messages, # compatibility when use openai-like endpoint
- )
+ try:
+ token_request: Final = _google_count_tokens_request(model_name, data)
+ except ValidationError as error:
+ raise HTTPException(status_code=400, detail="Invalid Google countTokens request body") from error
# Call the internal token counter function with direct request flag set to False
token_response: Final = await internal_token_counter(
diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py
index 6b7d84070af..ece9d6be4b2 100644
--- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py
+++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py
@@ -456,6 +456,140 @@ def test_gemini_count_tokens_route_reports_the_runtime_total_for_a_model_bedrock
assert mantle.drain() == ()
+def test_runtime_404_on_claude_falls_back_to_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None:
+ gateway: Final = counting_proxy.gateway
+ system: Final = f"{_SYSTEM} status=404"
+ with (
+ wire_server(_runtime) as runtime,
+ wire_server(_mantle(), port=mantle_port) as mantle,
+ gateway.scenario() as scenario,
+ ):
+ model: Final = _deployment(scenario, runtime.url)
+ response: Final = _count(gateway, {**_bare(model), "system": system, "tools": _TOOLS})
+ assert _payload(response) == {"input_tokens": _MANTLE_COUNT}, response.text
+ assert len(_runtime_count_targets(runtime)) == 1
+ assert _mantle_bodies(mantle) == (_mantle_body(system=system, tools=_TOOLS),)
+
+
+def test_runtime_404_and_mantle_404_surface_provider_error(counting_proxy: OwnedProxy, mantle_port: int) -> None:
+ gateway: Final = counting_proxy.gateway
+ system: Final = f"{_SYSTEM} status=404"
+ with (
+ wire_server(_runtime) as runtime,
+ wire_server(_mantle(_rejecting(404)), port=mantle_port) as mantle,
+ gateway.scenario() as scenario,
+ ):
+ model: Final = _deployment(scenario, runtime.url)
+ response: Final = _count(gateway, {**_bare(model), "system": system, "tools": _TOOLS})
+ assert response.status_code == 404, response.text
+ assert _REJECTION in response.text
+ assert len(_runtime_count_targets(runtime)) == 1
+ assert _mantle_bodies(mantle) == (_mantle_body(system=system, tools=_TOOLS),)
+
+
+def test_runtime_404_on_nova_does_not_call_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None:
+ gateway: Final = counting_proxy.gateway
+ with (
+ wire_server(_runtime) as runtime,
+ wire_server(_mantle(), port=mantle_port) as mantle,
+ gateway.scenario() as scenario,
+ ):
+ model: Final = _deployment(scenario, runtime.url, _NOVA)
+ body: Final = {**_bare(model), "messages": [{"role": "user", "content": "status=404"}]}
+ response: Final = _count(gateway, body)
+ assert response.status_code == 404, response.text
+ assert len(_runtime_count_targets(runtime)) == 1
+ assert mantle.drain() == ()
+
+
+def test_google_count_tokens_forwards_system_and_tools_to_mantle(
+ counting_proxy: OwnedProxy, mantle_port: int
+) -> None:
+ gateway: Final = counting_proxy.gateway
+ system_text: Final = "Use the provided tool"
+ tool_schema: Final = {"type": "object", "properties": {"location": {"type": "string"}}}
+ body: Final[dict[str, JsonValue]] = {
+ "contents": [{"role": "user", "parts": [{"text": "status=404"}]}],
+ "systemInstruction": {"parts": [{"text": system_text}]},
+ "tools": [
+ {
+ "functionDeclarations": [
+ {"name": "get_weather", "description": "Look up weather", "parameters": tool_schema}
+ ]
+ }
+ ],
+ }
+ with (
+ wire_server(_runtime) as runtime,
+ wire_server(_mantle(), port=mantle_port) as mantle,
+ gateway.scenario() as scenario,
+ ):
+ model: Final = _deployment(scenario, runtime.url)
+ response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", body)
+ assert _payload(response) == {"totalTokens": _MANTLE_COUNT, "promptTokensDetails": []}, response.text
+ assert len(_runtime_count_targets(runtime)) == 1
+ assert _mantle_bodies(mantle) == (
+ _mantle_body(
+ messages=[{"role": "user", "content": "status=404"}],
+ system=[{"type": "text", "text": system_text}],
+ tools=[
+ {
+ "name": "get_weather",
+ "description": "Look up weather",
+ "input_schema": tool_schema,
+ }
+ ],
+ ),
+ )
+
+
+def test_google_count_tokens_normalizes_uppercase_parameters_for_mantle(
+ counting_proxy: OwnedProxy, mantle_port: int
+) -> None:
+ gateway: Final = counting_proxy.gateway
+ body: Final[dict[str, JsonValue]] = {
+ "contents": [{"role": "user", "parts": [{"text": "status=404"}]}],
+ "tools": [
+ {
+ "functionDeclarations": [
+ {
+ "name": "get_weather",
+ "parameters": {
+ "type": "OBJECT",
+ "properties": {"location": {"type": "STRING"}},
+ "required": ["location"],
+ },
+ }
+ ]
+ }
+ ],
+ }
+ with (
+ wire_server(_runtime) as runtime,
+ wire_server(_mantle(), port=mantle_port) as mantle,
+ gateway.scenario() as scenario,
+ ):
+ model: Final = _deployment(scenario, runtime.url)
+ response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", body)
+ assert _payload(response) == {"totalTokens": _MANTLE_COUNT, "promptTokensDetails": []}, response.text
+ assert len(_runtime_count_targets(runtime)) == 1
+ assert _mantle_bodies(mantle) == (
+ _mantle_body(
+ messages=[{"role": "user", "content": "status=404"}],
+ tools=[
+ {
+ "name": "get_weather",
+ "input_schema": {
+ "type": "object",
+ "properties": {"location": {"type": "string"}},
+ "required": ["location"],
+ },
+ }
+ ],
+ ),
+ )
+
+
def test_responses_input_tokens_counts_through_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None:
gateway: Final = counting_proxy.gateway
with (
diff --git a/tests/integration/providers/test_openai_count_tokens_tools_wire.py b/tests/integration/providers/test_openai_count_tokens_tools_wire.py
new file mode 100644
index 00000000000..bfae84e333d
--- /dev/null
+++ b/tests/integration/providers/test_openai_count_tokens_tools_wire.py
@@ -0,0 +1,161 @@
+import json
+from collections.abc import Iterator
+from typing import Final
+
+import pytest
+from integration._support.client import Scenario, gateway_from_environment
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
+
+_COUNT: Final = 4242
+_TOOL_SCHEMA: Final[dict[str, JsonValue]] = {"type": "object", "properties": {}}
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+
+
+@pytest.fixture(scope="module")
+def counting_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
+ with (
+ gateway_from_environment() as gateway,
+ owned_proxy_process(gateway, tmp_path_factory.mktemp("openai-count-tools"), {}, workers=1) as owned,
+ ):
+ yield owned
+
+
+def _deployment(scenario: Scenario, api_base: str) -> str:
+ return scenario.model(
+ model="openai/gpt-4o-mini",
+ api_base=f"{api_base}/v1",
+ api_key="sk-integration-openai",
+ )
+
+
+def _count_response(_request: Request) -> Reply:
+ return Reply(body=json.dumps({"input_tokens": _COUNT}).encode())
+
+
+def _count_body(wire: Wire) -> dict[str, JsonValue]:
+ requests: Final = wire.drain()
+ assert len(requests) == 1, requests
+ request: Final = requests[0]
+ assert request.method == "POST"
+ assert request.target == "/v1/responses/input_tokens", request.target
+ return _JSON_OBJECT.validate_json(request.body)
+
+
+def test_openai_count_tokens_uses_deployment_api_base(counting_proxy: OwnedProxy) -> None:
+ gateway: Final = counting_proxy.gateway
+ with wire_server(_count_response) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(scenario, wire.url)
+ response: Final = gateway.request(
+ "POST",
+ "/v1/messages/count_tokens",
+ {"model": model, "messages": [{"role": "user", "content": "hello"}]},
+ )
+ assert response.status_code == 200, response.text
+ assert _JSON_OBJECT.validate_json(response.content)["input_tokens"] == _COUNT
+ assert "tools" not in _count_body(wire)
+
+
+def test_google_count_tokens_maps_anthropic_tools_to_openai(counting_proxy: OwnedProxy) -> None:
+ gateway: Final = counting_proxy.gateway
+ with wire_server(_count_response) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(scenario, wire.url)
+ response: Final = gateway.request(
+ "POST",
+ f"/v1beta/models/{model}:countTokens",
+ {
+ "contents": [{"role": "user", "parts": [{"text": "hello"}]}],
+ "tools": [
+ {
+ "functionDeclarations": [
+ {
+ "name": "lookup",
+ "description": "Find a record",
+ "parameters": _TOOL_SCHEMA,
+ }
+ ]
+ }
+ ],
+ },
+ )
+ assert response.status_code == 200, response.text
+ assert _JSON_OBJECT.validate_json(response.content)["totalTokens"] == _COUNT
+ assert _count_body(wire)["tools"] == [
+ {
+ "type": "function",
+ "name": "lookup",
+ "description": "Find a record",
+ "parameters": _TOOL_SCHEMA,
+ "strict": False,
+ }
+ ]
+
+
+def test_messages_count_tokens_maps_anthropic_tools_to_openai(counting_proxy: OwnedProxy) -> None:
+ gateway: Final = counting_proxy.gateway
+ with wire_server(_count_response) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(scenario, wire.url)
+ response: Final = gateway.request(
+ "POST",
+ "/v1/messages/count_tokens",
+ {
+ "model": model,
+ "messages": [{"role": "user", "content": "hello"}],
+ "tools": [
+ {
+ "name": "lookup",
+ "description": "Find a record",
+ "input_schema": _TOOL_SCHEMA,
+ }
+ ],
+ },
+ )
+ assert response.status_code == 200, response.text
+ assert _JSON_OBJECT.validate_json(response.content)["input_tokens"] == _COUNT
+ assert _count_body(wire)["tools"] == [
+ {
+ "type": "function",
+ "name": "lookup",
+ "description": "Find a record",
+ "parameters": _TOOL_SCHEMA,
+ "strict": False,
+ }
+ ]
+
+
+def test_utils_token_counter_keeps_chat_tool_conversion(counting_proxy: OwnedProxy) -> None:
+ gateway: Final = counting_proxy.gateway
+ with wire_server(_count_response) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(scenario, wire.url)
+ response: Final = gateway.request(
+ "POST",
+ "/utils/token_counter",
+ {
+ "model": model,
+ "messages": [{"role": "user", "content": "hello"}],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "lookup",
+ "description": "Find a record",
+ "parameters": _TOOL_SCHEMA,
+ },
+ }
+ ],
+ },
+ params={"call_endpoint": "true"},
+ )
+ assert response.status_code == 200, response.text
+ assert _JSON_OBJECT.validate_json(response.content)["total_tokens"] == _COUNT
+ assert _count_body(wire)["tools"] == [
+ {
+ "type": "function",
+ "name": "lookup",
+ "description": "Find a record",
+ "parameters": _TOOL_SCHEMA,
+ }
+ ]
diff --git a/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py b/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py
new file mode 100644
index 00000000000..1a148b19f8d
--- /dev/null
+++ b/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py
@@ -0,0 +1,462 @@
+import asyncio
+import json
+import threading
+from collections.abc import Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import ExitStack
+from types import MappingProxyType
+from typing import Final
+from uuid import uuid4
+
+import anthropic
+import httpx
+import pytest
+from integration._support.client import Gateway, Scenario
+from integration._support.process import graceful_stop_seconds
+from integration._support.vertex import service_account_json
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
+
+_MODEL: Final = "gemini-4-argon"
+_PROJECT: Final = "scripted-project"
+_LOCATION: Final = "us-east5"
+_COUNT: Final = 777
+_UNIT: Final = " red blue green yellow orange purple black white silver gold"
+_USER: Final = f"{_UNIT * 100}\nReturn exactly OK."
+_MESSAGES: Final[JsonValue] = [{"role": "user", "content": _USER}]
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_JSON_LIST: Final = TypeAdapter(list[JsonValue])
+_REPLY: Final[Mapping[str, JsonValue]] = {
+ "candidates": [
+ {
+ "content": {"role": "model", "parts": [{"text": "OK"}]},
+ "finishReason": "STOP",
+ }
+ ],
+ "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 1, "totalTokenCount": 11},
+}
+
+
+class _GeminiPeer:
+ def __init__(self) -> None:
+ self.status: int = 200
+ self.requests: tuple[Request, ...] = ()
+ self._lock: Final = threading.Lock()
+
+ def set_status(self, status: int) -> None:
+ self.status = status
+
+ def __call__(self, request: Request) -> Reply:
+ with self._lock:
+ self.requests = (*self.requests, request)
+ status: Final = self.status
+ if request.target.endswith(":countTokens"):
+ return Reply(status=status, body=json.dumps({"totalTokens": _COUNT}).encode())
+ if request.target.endswith(":generateContent"):
+ return Reply(body=json.dumps(_REPLY).encode())
+ return Reply(status=404, body=json.dumps({"error": request.target}).encode())
+
+ def drain(self) -> tuple[Request, ...]:
+ with self._lock:
+ requests: Final = self.requests
+ self.requests = ()
+ return requests
+
+
+def _deployment(gateway: Gateway, scenario: Scenario, api_base: str) -> str:
+ return scenario.model(
+ model=f"vertex_ai/{_MODEL}",
+ api_base=api_base,
+ api_key=None,
+ vertex_project=_PROJECT,
+ vertex_location=_LOCATION,
+ vertex_credentials=service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")),
+ )
+
+
+def _messages(model: str, system: JsonValue | None, tools: JsonValue | None) -> Mapping[str, JsonValue]:
+ return {
+ "model": model,
+ "messages": _MESSAGES,
+ **({"system": system} if system is not None else {}),
+ **({"tools": tools} if tools is not None else {}),
+ }
+
+
+def _json_body(request: Request) -> Mapping[str, JsonValue]:
+ return _JSON_OBJECT.validate_json(request.body)
+
+
+def _count_requests(wire: Wire) -> tuple[Request, ...]:
+ return tuple(request for request in wire.drain() if request.target.endswith(":countTokens"))
+
+
+def _count_tokens_tools(generated_tools: JsonValue) -> JsonValue:
+ return _JSON_LIST.validate_python(
+ [
+ {
+ {
+ "function_declarations": "functionDeclarations",
+ "url_context": "urlContext",
+ "code_execution": "codeExecution",
+ }.get(key, key): value
+ for key, value in tool.items()
+ }
+ for tool in TypeAdapter(list[Mapping[str, JsonValue]]).validate_python(generated_tools)
+ ]
+ )
+
+
+def _count_tokens_request_from_generate_content(
+ generated_body: Mapping[str, JsonValue],
+) -> Mapping[str, JsonValue]:
+ system_instruction: Final = generated_body.get("systemInstruction", generated_body.get("system_instruction"))
+ generated_tools: Final = generated_body.get("tools")
+ return MappingProxyType(
+ {
+ "contents": generated_body["contents"],
+ **({"systemInstruction": system_instruction} if system_instruction is not None else {}),
+ **({"tools": _count_tokens_tools(generated_tools)} if generated_tools is not None else {}),
+ }
+ )
+
+
+def _tools(description: str) -> JsonValue:
+ return _JSON_LIST.validate_python(
+ [
+ {
+ "name": "noop",
+ "description": description,
+ "input_schema": {"type": "object", "properties": {}},
+ }
+ ]
+ )
+
+
+@pytest.mark.parametrize(
+ ("large_system", "large_tool"),
+ ((False, False), (True, False), (False, True), (True, True)),
+ ids=("baseline", "large-system", "large-tool", "large-system-and-tool"),
+)
+def test_messages_and_count_tokens_send_matching_gemini_context(
+ gateway: Gateway, large_system: bool, large_tool: bool
+) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ system: Final = (
+ f"{uuid4().hex}\n"
+ "This is a synthetic context-budget diagnostic. Return exactly OK." + (_UNIT * 100 if large_system else "")
+ if large_system
+ else None
+ )
+ tools: Final = _tools(_UNIT * 100) if large_tool else None
+ body: Final = _messages(model, system, tools)
+ generated: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 16})
+ counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body)
+ assert generated.status_code == 200, generated.text
+ assert counted.status_code == 200, counted.text
+ assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _COUNT, counted.text
+ requests: Final = peer.drain()
+ generated_body: Final = _json_body(
+ next(request for request in requests if request.target.endswith(":generateContent"))
+ )
+ count_body: Final = _json_body(next(request for request in requests if request.target.endswith(":countTokens")))
+ assert count_body == _count_tokens_request_from_generate_content(generated_body)
+
+
+def test_count_tokens_keeps_each_text_block_in_system_instruction(gateway: Gateway) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ system: Final[JsonValue] = [
+ {"type": "text", "text": "First instruction"},
+ {"type": "text", "text": "Second instruction"},
+ ]
+ response: Final = gateway.request("POST", "/v1/messages/count_tokens", _messages(model, system, None))
+ assert response.status_code == 200, response.text
+ requests: Final = _count_requests(wire)
+ assert len(requests) == 1
+ assert _json_body(requests[0]) == {
+ "contents": [{"role": "user", "parts": [{"text": _USER}]}],
+ "systemInstruction": {"parts": [{"text": "First instruction"}, {"text": "Second instruction"}]},
+ }
+
+
+def test_count_tokens_without_system_or_tools_sends_contents_only(gateway: Gateway) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ response: Final = gateway.request("POST", "/v1/messages/count_tokens", _messages(model, None, None))
+ assert response.status_code == 200, response.text
+ requests: Final = _count_requests(wire)
+ assert len(requests) == 1
+ assert _json_body(requests[0]) == {"contents": [{"role": "user", "parts": [{"text": _USER}]}]}
+
+
+_SDK_COUNT_ARGS: Final[Mapping[str, JsonValue]] = MappingProxyType(
+ {
+ "messages": [{"role": "user", "content": _USER}],
+ "system": "Follow the system instruction.",
+ "tools": [
+ {
+ "name": "noop",
+ "description": "Return no output",
+ "input_schema": {"type": "object", "properties": {}},
+ }
+ ],
+ }
+)
+
+
+def _sync_sdk_count_tokens(gateway: Gateway, model: str) -> int:
+ client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
+ return client.messages.count_tokens(model=model, **_SDK_COUNT_ARGS).input_tokens # pyright: ignore[reportArgumentType] # SDK TypedDicts reject a JsonValue mapping
+
+
+async def _async_sdk_count_tokens(gateway: Gateway, model: str) -> int:
+ client: Final = anthropic.AsyncAnthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
+ return (await client.messages.count_tokens(model=model, **_SDK_COUNT_ARGS)).input_tokens # pyright: ignore[reportArgumentType] # SDK TypedDicts reject a JsonValue mapping
+
+
+@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync-sdk", "async-sdk"))
+def test_anthropic_sdk_count_tokens_forwards_system_and_tools(gateway: Gateway, asynchronous: bool) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ input_tokens: Final = (
+ asyncio.run(_async_sdk_count_tokens(gateway, model))
+ if asynchronous
+ else _sync_sdk_count_tokens(gateway, model)
+ )
+ assert input_tokens == _COUNT
+ requests: Final = _count_requests(wire)
+ assert len(requests) == 1
+ request_body: Final = _json_body(requests[0])
+ assert request_body["systemInstruction"] == {"parts": [{"text": "Follow the system instruction."}]}
+ assert request_body["tools"] == [
+ {
+ "functionDeclarations": [
+ {"name": "noop", "description": "Return no output", "parameters": {"type": "object"}}
+ ]
+ }
+ ]
+
+
+def test_google_count_tokens_forwards_system_and_tools_to_vertex(gateway: Gateway) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ body: Final[Mapping[str, JsonValue]] = {
+ "contents": [
+ {"role": "user", "parts": [{"text": _USER}]},
+ {"role": "model", "parts": [{"functionCall": {"name": "noop", "args": {"query": "value"}}}]},
+ {
+ "role": "user",
+ "parts": [{"functionResponse": {"name": "noop", "response": {"result": "ok"}}}],
+ },
+ ],
+ "systemInstruction": {"parts": [{"text": "Follow this instruction."}]},
+ "tools": [
+ {
+ "functionDeclarations": [
+ {
+ "name": "noop",
+ "description": "Return no output",
+ "parameters": {"type": "OBJECT", "properties": {}},
+ }
+ ]
+ }
+ ],
+ }
+ response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", body)
+ assert response.status_code == 200, response.text
+ assert _JSON_OBJECT.validate_json(response.content)["totalTokens"] == _COUNT, response.text
+ requests: Final = _count_requests(wire)
+ assert len(requests) == 1
+ request_body: Final = _json_body(requests[0])
+ assert request_body["contents"] == body["contents"]
+ assert request_body["systemInstruction"] == body["systemInstruction"]
+ assert request_body["tools"] == [
+ {
+ "functionDeclarations": [
+ {
+ "name": "noop",
+ "description": "Return no output",
+ "parameters": {"type": "object"},
+ }
+ ]
+ }
+ ]
+
+
+def test_responses_input_tokens_maps_flat_function_tools_to_vertex(gateway: Gateway) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ body: Final[Mapping[str, JsonValue]] = {
+ "model": model,
+ "instructions": "Follow the system instruction.",
+ "input": [
+ {
+ "type": "function_call",
+ "call_id": "call_lookup",
+ "name": "lookup",
+ "arguments": '{"query":"value"}',
+ },
+ {
+ "type": "function_call_output",
+ "call_id": "call_lookup",
+ "output": "result",
+ },
+ ],
+ "tools": [
+ {
+ "type": "function",
+ "name": "lookup",
+ "description": "Look up a value",
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ],
+ }
+ response: Final = gateway.request("POST", "/v1/responses/input_tokens", body)
+ assert response.status_code == 200, response.text
+ assert _JSON_OBJECT.validate_json(response.content)["input_tokens"] == _COUNT, response.text
+ requests: Final = _count_requests(wire)
+ assert len(requests) == 1
+ request_body: Final = _json_body(requests[0])
+ assert request_body["tools"] == [
+ {
+ "functionDeclarations": [
+ {
+ "name": "lookup",
+ "description": "Look up a value",
+ "parameters": {"type": "object"},
+ }
+ ]
+ }
+ ]
+
+
+def test_utils_token_counter_maps_openai_function_tools_to_vertex(gateway: Gateway) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ tool_schema: Final[JsonValue] = {"type": "object", "properties": {}}
+ body: Final[Mapping[str, JsonValue]] = {
+ "model": model,
+ "messages": _MESSAGES,
+ "system": "Follow the system instruction.",
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "noop",
+ "description": "Return no output",
+ "parameters": tool_schema,
+ },
+ }
+ ],
+ }
+ response: Final = gateway.request("POST", "/utils/token_counter", body, params={"call_endpoint": "true"})
+ assert response.status_code == 200, response.text
+ assert _JSON_OBJECT.validate_json(response.content)["total_tokens"] == _COUNT, response.text
+ requests: Final = _count_requests(wire)
+ assert len(requests) == 1
+ request_body: Final = _json_body(requests[0])
+ assert request_body["systemInstruction"] == {"parts": [{"text": "Follow the system instruction."}]}
+ assert request_body["tools"] == [
+ {
+ "functionDeclarations": [
+ {
+ "name": "noop",
+ "description": "Return no output",
+ "parameters": {"type": "object"},
+ }
+ ]
+ }
+ ]
+
+
+def test_count_tokens_surfaces_peer_errors_and_rejects_non_text_system(gateway: Gateway) -> None:
+ peer: Final = _GeminiPeer()
+ peer.set_status(400)
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ response: Final = gateway.request("POST", "/v1/messages/count_tokens", _messages(model, None, None))
+ assert response.status_code == 400, response.text
+ assert len(_count_requests(wire)) == 1
+
+ invalid_peer: Final = _GeminiPeer()
+ with wire_server(invalid_peer) as invalid_wire, gateway.scenario() as scenario:
+ invalid_model: Final = _deployment(gateway, scenario, invalid_wire.url)
+ invalid_response: Final = gateway.request(
+ "POST",
+ "/v1/messages/count_tokens",
+ _messages(
+ invalid_model,
+ [{"type": "image", "source": {"type": "base64", "data": "AAAA"}}],
+ None,
+ ),
+ )
+ assert invalid_response.status_code == 400, invalid_response.text
+ assert _count_requests(invalid_wire) == ()
+
+
+@pytest.mark.parametrize(
+ "tools",
+ (
+ [{"name": "noop", "input_schema": {"type": "object", "properties": {"query": "not-a-schema"}}}],
+ [{"name": "noop", "input_schema": "not-a-schema"}],
+ ),
+ ids=("malformed-property", "string-schema"),
+)
+def test_count_tokens_rejects_unconvertible_tools_without_request(gateway: Gateway, tools: JsonValue) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ response: Final = gateway.request("POST", "/v1/messages/count_tokens", _messages(model, None, tools))
+ assert response.status_code == 400, response.text
+ assert "tools" in response.text
+ assert _count_requests(wire) == ()
+
+
+def _count_wave(clients: Sequence[httpx.Client], body: Mapping[str, JsonValue]) -> tuple[tuple[int, JsonValue], ...]:
+ def count(client: httpx.Client) -> tuple[int, JsonValue]:
+ response: Final = client.post("/v1/messages/count_tokens", json=dict(body))
+ payload: Final = _JSON_OBJECT.validate_json(response.content)
+ return response.status_code, payload.get("input_tokens")
+
+ with ThreadPoolExecutor(max_workers=len(clients)) as executor:
+ return tuple(executor.map(count, clients))
+
+
+def test_concurrent_count_waves_recover_after_vertex_count_outage(gateway: Gateway) -> None:
+ peer: Final = _GeminiPeer()
+ with wire_server(peer) as wire, gateway.scenario() as scenario, ExitStack() as stack:
+ model: Final = _deployment(gateway, scenario, wire.url)
+ clients: Final = tuple(
+ stack.enter_context(
+ httpx.Client(
+ base_url=str(gateway.client.base_url),
+ headers={"Authorization": f"Bearer {gateway.key}"},
+ timeout=30,
+ trust_env=False,
+ )
+ )
+ for _ in range(3)
+ )
+ body: Final = _messages(model, None, None)
+ assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT))
+ peer.set_status(503)
+ assert _count_wave(clients, body) == ((503, None), (503, None), (503, None))
+ peer.set_status(200)
+ assert _count_wave(clients, body) == ((200, _COUNT), (200, _COUNT), (200, _COUNT))
+ requests: Final = _count_requests(wire)
+ assert len(requests) == 9
+ assert (
+ tuple(_json_body(request) for request in requests)
+ == ({"contents": [{"role": "user", "parts": [{"text": _USER}]}]},) * 9
+ )
diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py
index a3dd8021f73..8a21027872a 100644
--- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py
+++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py
@@ -93,8 +93,10 @@ async def test_bedrock_runtime_count_is_kept_when_it_answers() -> None:
@pytest.mark.parametrize(
("model_to_use", "runtime"),
(
+ ("global.anthropic.claude-opus-4-8", (401, {"Message": "not authorized to perform: bedrock:CountTokens"})),
("global.anthropic.claude-opus-4-8", (403, {"Message": "not authorized to perform: bedrock:CountTokens"})),
("amazon.nova-pro-v1:0", (400, UNSUPPORTED)),
+ ("amazon.nova-pro-v1:0", (404, UNSUPPORTED)),
),
)
async def test_other_bedrock_runtime_failures_are_not_retried_on_mantle(
@@ -120,7 +122,7 @@ async def test_other_bedrock_runtime_failures_are_not_retried_on_mantle(
@pytest.mark.asyncio
async def test_mantle_failure_is_reported_with_its_status() -> None:
bedrock: Final = _Bedrock(
- runtime=(400, UNSUPPORTED),
+ runtime=(404, UNSUPPORTED),
mantle=(404, {"type": "error", "error": {"type": "not_found_error", "message": "does not exist"}}),
)
@@ -138,3 +140,37 @@ async def test_mantle_failure_is_reported_with_its_status() -> None:
assert result.tokenizer_type == "bedrock_mantle_api"
assert "does not exist" in (result.error_message or "")
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
+
+
+@pytest.mark.asyncio
+async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> None:
+ bedrock: Final = _Bedrock(runtime=(404, UNSUPPORTED), mantle=(200, {"input_tokens": 2177}))
+ tools: Final = [
+ {
+ "name": "lookup",
+ "description": "Find a record",
+ "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}},
+ }
+ ]
+
+ result: Final = await _counter(bedrock).count_tokens(
+ model_to_use="global.anthropic.claude-opus-4-8",
+ messages=MESSAGES,
+ contents=None,
+ deployment=DEPLOYMENT,
+ request_model="claude-opus-4-8",
+ system="You are a terse assistant.",
+ tools=tools,
+ )
+
+ assert result is not None
+ assert result.error is False
+ assert result.total_tokens == 2177
+ assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
+ mantle_body: Final = _JSON_BODY.validate_json(bedrock.requests[1].content)
+ assert mantle_body == {
+ "model": "anthropic.claude-opus-4-8",
+ "messages": MESSAGES,
+ "system": "You are a terse assistant.",
+ "tools": tools,
+ }
diff --git a/tests/unit/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py
index 70946e10590..9a47d01ec46 100644
--- a/tests/unit/llms/gemini/test_gemini_common_utils.py
+++ b/tests/unit/llms/gemini/test_gemini_common_utils.py
@@ -1,8 +1,149 @@
+from typing import Final
from unittest.mock import AsyncMock, patch
+import httpx
import pytest
+from pydantic import JsonValue, TypeAdapter
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.gemini.common_utils import GeminiModelInfo, GoogleAIStudioTokenCounter
+from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler
+from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter as GeminiCountTokensHandler
+
+_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
+
+
+class _GeminiCountTokensPeer:
+ def __init__(self) -> None:
+ self.request_body: dict[str, JsonValue] | None = None
+
+ def __call__(self, request: httpx.Request) -> httpx.Response:
+ self.request_body = _JSON_BODY.validate_json(request.content)
+ return httpx.Response(200, json={"totalTokens": 11}, request=request)
+
+
+@pytest.mark.asyncio
+async def test_google_ai_studio_count_tokens_forwards_system_and_tools() -> None:
+ peer: Final = _GeminiCountTokensPeer()
+ client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(peer))
+
+ with patch.object(gemini_count_tokens_handler, "get_async_httpx_client", return_value=client):
+ result: Final = await GoogleAIStudioTokenCounter().count_tokens(
+ model_to_use="gemini-2.5-flash",
+ messages=None,
+ contents=[{"role": "user", "parts": [{"text": "hello"}]}],
+ deployment={
+ "litellm_params": {
+ "api_key": "test-key",
+ "api_base": "https://count-tokens.test",
+ "system_instruction": {"parts": [{"text": "deployment-level"}]},
+ }
+ },
+ request_model="gemini/gemini-2.5-flash",
+ tools=[
+ {
+ "name": "lookup",
+ "description": "Find a record",
+ "input_schema": {"type": "object", "properties": {}},
+ }
+ ],
+ system="You are terse.",
+ )
+
+ assert result is not None
+ assert result.error is False
+ assert result.total_tokens == 11
+ assert peer.request_body == {
+ "generateContentRequest": {
+ "model": "models/gemini-2.5-flash",
+ "contents": [{"role": "user", "parts": [{"text": "hello"}]}],
+ "systemInstruction": {"parts": [{"text": "You are terse."}]},
+ "tools": [
+ {
+ "functionDeclarations": [
+ {
+ "name": "lookup",
+ "description": "Find a record",
+ "parameters": {"type": "object"},
+ }
+ ]
+ }
+ ],
+ }
+ }
+
+
+@pytest.mark.asyncio
+async def test_google_ai_studio_count_tokens_rejects_image_system_without_request() -> None:
+ peer: Final = _GeminiCountTokensPeer()
+ client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(peer))
+
+ with patch.object(gemini_count_tokens_handler, "get_async_httpx_client", return_value=client):
+ result: Final = await GoogleAIStudioTokenCounter().count_tokens(
+ model_to_use="gemini-2.5-flash",
+ messages=None,
+ contents=[{"role": "user", "parts": [{"text": "hello"}]}],
+ deployment={"litellm_params": {"api_key": "test-key", "api_base": "https://count-tokens.test"}},
+ request_model="gemini/gemini-2.5-flash",
+ system=[
+ {"type": "text", "text": "You are terse."},
+ {
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="},
+ },
+ ],
+ )
+
+ assert result is not None
+ assert result.error is True
+ assert result.status_code == 400
+ assert peer.request_body is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("system_instruction", "tools", "expected_body"),
+ (
+ (
+ {"parts": [{"text": "Follow the system rule."}]},
+ [{"functionDeclarations": [{"name": "lookup", "parameters": {"type": "OBJECT"}}]}],
+ {
+ "generateContentRequest": {
+ "model": "models/gemini-4-argon",
+ "contents": [{"role": "user", "parts": [{"text": "hello"}]}],
+ "systemInstruction": {"parts": [{"text": "Follow the system rule."}]},
+ "tools": [{"functionDeclarations": [{"name": "lookup", "parameters": {"type": "OBJECT"}}]}],
+ }
+ },
+ ),
+ (
+ None,
+ [],
+ {"contents": [{"role": "user", "parts": [{"text": "hello"}]}]},
+ ),
+ ),
+)
+async def test_gemini_count_tokens_sends_optional_request_fields(
+ system_instruction: dict[str, JsonValue] | None,
+ tools: list[dict[str, JsonValue]],
+ expected_body: dict[str, JsonValue],
+) -> None:
+ peer: Final = _GeminiCountTokensPeer()
+ token_counter: Final = GeminiCountTokensHandler(
+ client=AsyncHTTPHandler(transport=httpx.MockTransport(peer))
+ )
+
+ result: Final = await token_counter.acount_tokens(
+ contents=[{"role": "user", "parts": [{"text": "hello"}]}],
+ model="gemini-4-argon",
+ api_key="test-key",
+ api_base="https://count-tokens.test",
+ system_instruction=system_instruction,
+ tools=tools,
+ )
+
+ assert result == {"totalTokens": 11}
+ assert peer.request_body == expected_body
class TestGeminiModelInfo:
@@ -158,7 +299,10 @@ class TestGoogleAIStudioTokenCounter:
# Verify the mock was called correctly
mock_acount_tokens.assert_called_once_with(
- model=model_to_use, contents=contents
+ model=model_to_use,
+ contents=contents,
+ system_instruction=None,
+ tools=None,
)
def test_clean_contents_for_gemini_api_removes_id_field(self):
diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py
index c2efc1acdb9..98ddfa08018 100644
--- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py
+++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py
@@ -1,3 +1,4 @@
+from typing import Final
import pytest
@@ -79,6 +80,54 @@ def test_transform_includes_tools():
assert result["tools"] == tools
+_SCHEMA: Final[dict[str, object]] = {"type": "object", "properties": {}} # mutable-ok: matches the tools param type
+
+
+@pytest.mark.parametrize(
+ ("tool", "expected"),
+ (
+ (
+ {"name": "lookup", "input_schema": _SCHEMA},
+ {"type": "function", "name": "lookup", "description": "", "parameters": _SCHEMA, "strict": False},
+ ),
+ (
+ {"name": "lookup", "input_schema": _SCHEMA, "strict": True},
+ {"type": "function", "name": "lookup", "description": "", "parameters": _SCHEMA, "strict": True},
+ ),
+ (
+ {"type": "custom", "name": "lookup", "description": "Find a record", "input_schema": _SCHEMA},
+ {
+ "type": "function",
+ "name": "lookup",
+ "description": "Find a record",
+ "parameters": _SCHEMA,
+ "strict": False,
+ },
+ ),
+ ),
+ ids=("anthropic", "anthropic-strict", "anthropic-custom-type"),
+)
+def test_transform_anthropic_tool_shapes(tool: dict[str, object], expected: dict[str, object]) -> None:
+ result: Final = OpenAICountTokensConfig().transform_request_to_count_tokens(
+ model="gpt-4o", input="Find a record", tools=[tool]
+ )
+
+ assert result["tools"] == [expected]
+
+
+def test_transform_builtin_tool_passes_through():
+ config: Final = OpenAICountTokensConfig()
+ tools: Final = [{"type": "web_search", "search_context_size": "low"}]
+
+ result: Final = config.transform_request_to_count_tokens(
+ model="gpt-4o",
+ input="Find a record",
+ tools=tools,
+ )
+
+ assert result["tools"] == tools
+
+
def test_transform_no_instructions_no_tools():
"""Test that None values are not included."""
config = OpenAICountTokensConfig()
@@ -102,9 +151,7 @@ def test_messages_to_responses_input_basic():
{"role": "user", "content": "How are you?"},
]
- input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
- messages
- )
+ input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert len(input_items) == 3
assert input_items[0] == {"role": "user", "content": "Hello"}
@@ -120,9 +167,7 @@ def test_messages_to_responses_input_with_system():
{"role": "user", "content": "Hello"},
]
- input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
- messages
- )
+ input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert len(input_items) == 1
assert input_items[0] == {"role": "user", "content": "Hello"}
@@ -136,9 +181,7 @@ def test_messages_to_responses_input_with_developer():
{"role": "user", "content": "Hello"},
]
- input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
- messages
- )
+ input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert len(input_items) == 1
assert instructions == "Be concise."
@@ -151,9 +194,7 @@ def test_messages_to_responses_input_with_tool():
{"role": "tool", "content": "72°F", "tool_call_id": "call_123"},
]
- input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
- messages
- )
+ input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
assert len(input_items) == 2
assert input_items[1] == {
@@ -424,10 +465,7 @@ def test_validate_request_missing_input():
def test_get_endpoint_default():
"""Test default endpoint URL."""
config = OpenAICountTokensConfig()
- assert (
- config.get_openai_count_tokens_endpoint()
- == "https://api.openai.com/v1/responses/input_tokens"
- )
+ assert config.get_openai_count_tokens_endpoint() == "https://api.openai.com/v1/responses/input_tokens"
def test_get_endpoint_custom_base():
diff --git a/tests/unit/llms/vertex_ai/count_tokens/__init__.py b/tests/unit/llms/vertex_ai/count_tokens/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py
new file mode 100644
index 00000000000..444920e1ecc
--- /dev/null
+++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py
@@ -0,0 +1,195 @@
+from typing import Final
+
+import pytest
+from pydantic import JsonValue
+
+from litellm.llms.anthropic.pass_through.adapters.transformation import (
+ LiteLLMAnthropicMessagesAdapter,
+ truncate_tool_name,
+)
+from litellm.llms.vertex_ai.count_tokens.context import (
+ VertexGeminiCountContext,
+ VertexGeminiCountContextError,
+ vertex_gemini_count_context,
+)
+from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
+
+_SCHEMA: Final[JsonValue] = {
+ "type": "object",
+ "properties": {},
+ "additionalProperties": False,
+}
+_FUNCTION_DECLARATION: Final = {
+ "functionDeclarations": [
+ {
+ "name": "lookup",
+ "description": "Look up a value",
+ "parameters": {"type": "object"},
+ }
+ ]
+}
+
+
+@pytest.mark.parametrize(
+ "tool",
+ (
+ {
+ "name": "lookup",
+ "description": "Look up a value",
+ "input_schema": _SCHEMA,
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "lookup",
+ "description": "Look up a value",
+ "parameters": _SCHEMA,
+ },
+ },
+ {
+ "type": "function",
+ "name": "lookup",
+ "description": "Look up a value",
+ "parameters": _SCHEMA,
+ },
+ ),
+ ids=("anthropic", "openai-chat", "responses"),
+)
+def test_function_tool_shapes_map_to_vertex_declarations(tool: JsonValue) -> None:
+ context: Final = vertex_gemini_count_context(system=None, tools=[tool])
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.tools == (_FUNCTION_DECLARATION,)
+ assert _SCHEMA["additionalProperties"] is False
+
+
+def test_long_anthropic_tool_name_is_truncated_for_vertex() -> None:
+ name: Final = "mcp__docs__" + "a" * 70
+ context: Final = vertex_gemini_count_context(
+ system=None,
+ tools=[{"name": name, "description": "Look up a value", "input_schema": _SCHEMA}],
+ )
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.tools == (
+ {
+ "functionDeclarations": [
+ {
+ "name": truncate_tool_name(name),
+ "description": "Look up a value",
+ "parameters": {"type": "object"},
+ }
+ ]
+ },
+ )
+ assert len(truncate_tool_name(name)) == 64
+
+
+def test_64_character_anthropic_tool_name_is_unchanged_for_vertex() -> None:
+ name: Final = "mcp__docs__" + "a" * 53
+ context: Final = vertex_gemini_count_context(
+ system=None,
+ tools=[{"name": name, "input_schema": _SCHEMA}],
+ )
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.tools == ({"functionDeclarations": [{"name": name, "parameters": {"type": "object"}}]},)
+
+
+def test_long_openai_chat_tool_name_is_unchanged_for_vertex() -> None:
+ name: Final = "mcp__docs__" + "a" * 70
+ context: Final = vertex_gemini_count_context(
+ system=None,
+ tools=[{"type": "function", "function": {"name": name, "parameters": _SCHEMA}}],
+ )
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.tools == ({"functionDeclarations": [{"name": name, "parameters": {"type": "object"}}]},)
+
+
+def test_unknown_tools_are_omitted() -> None:
+ context: Final = vertex_gemini_count_context(
+ system=None,
+ tools=[{"type": "web_search_preview"}, {"type": "computer_use"}],
+ )
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.tools is None
+
+
+def test_text_block_system_instruction_keeps_one_part_per_block() -> None:
+ context: Final = vertex_gemini_count_context(
+ system=[{"type": "text", "text": "First instruction."}, {"type": "text", "text": "Second instruction."}],
+ tools=None,
+ )
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.system_instruction == {"parts": ({"text": "First instruction."}, {"text": "Second instruction."})}
+
+
+def test_string_system_instruction_maps_to_one_part() -> None:
+ context: Final = vertex_gemini_count_context(system="Follow this instruction.", tools=None)
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.system_instruction == {"parts": ({"text": "Follow this instruction."},)}
+
+
+def test_non_text_system_block_returns_an_error_value() -> None:
+ context: Final = vertex_gemini_count_context(
+ system=[{"type": "image", "source": {"type": "base64", "data": "AAAA"}}],
+ tools=None,
+ )
+
+ assert isinstance(context, VertexGeminiCountContextError)
+ assert "text blocks" in context.error_message
+
+
+@pytest.mark.parametrize(("system", "tools"), ((None, None), ("", None), ([], [])))
+def test_empty_context_is_omitted(system: JsonValue | None, tools: JsonValue | None) -> None:
+ context: Final = vertex_gemini_count_context(system=system, tools=tools)
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.system_instruction is None
+ assert context.tools is None
+
+
+@pytest.mark.parametrize(
+ "tool",
+ (
+ {"description": "Look up a value", "input_schema": _SCHEMA},
+ {"type": "bash_20250124", "name": "bash"},
+ ),
+ ids=("unnamed", "anthropic-hosted"),
+)
+def test_anthropic_tools_count_the_same_declarations_inference_sends(tool: dict[str, JsonValue]) -> None:
+ translated_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai([tool]) # pyright: ignore[reportArgumentType] # raw Anthropic tool dict
+ inference_tools: Final = VertexGeminiConfig().map_function(
+ value=[dict(translated) for translated in translated_tools],
+ optional_params={},
+ )
+
+ context: Final = vertex_gemini_count_context(system=None, tools=[tool])
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.tools is not None
+ assert [tool_value["functionDeclarations"] for tool_value in context.tools] == [
+ inference_tool["function_declarations"] for inference_tool in inference_tools
+ ]
+
+
+@pytest.mark.parametrize(
+ "tools",
+ (
+ "lookup",
+ [1],
+ [{"name": "lookup", "input_schema": "not-a-schema"}],
+ [{"type": "function", "function": {"name": "lookup", "parameters": [1]}}],
+ [{"name": "lookup", "input_schema": {"type": "object", "properties": {"query": "not-a-schema"}}}],
+ ),
+ ids=("not-a-list", "non-object-tool", "string-schema", "list-parameters", "malformed-property"),
+)
+def test_invalid_tools_return_an_error_value(tools: JsonValue) -> None:
+ context: Final = vertex_gemini_count_context(system=None, tools=tools)
+
+ assert isinstance(context, VertexGeminiCountContextError)
+ assert "tools" in context.error_message
diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py
index c0b1173db29..f7ebc3eab54 100644
--- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py
+++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py
@@ -1,11 +1,17 @@
import time
from collections.abc import Iterator
-from unittest.mock import patch
+from typing import Final
+from unittest.mock import AsyncMock, patch
+import httpx
import pytest
+from pydantic import JsonValue, TypeAdapter
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler
from litellm.llms.vertex_ai.common_utils import (
+ VertexAITokenCounter,
convert_anyof_null_to_nullable,
convert_vertex_datetime_to_openai_datetime,
get_vertex_location_from_url,
@@ -17,6 +23,23 @@ from litellm.llms.vertex_ai.common_utils import (
validate_vertex_location,
vertex_request_labels_from_litellm_params,
)
+from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler
+
+
+@pytest.mark.asyncio
+async def test_vertex_gemini_rejects_non_text_system_blocks_before_request() -> None:
+ result: Final = await VertexAITokenCounter().count_tokens(
+ model_to_use="gemini-4-argon",
+ messages=[{"role": "user", "content": "Count this"}],
+ contents=None,
+ request_model="vertex_ai/gemini-4-argon",
+ system=[{"type": "image", "source": {"type": "base64", "data": "AAAA"}}],
+ )
+
+ assert result is not None
+ assert result.error is True
+ assert result.status_code == 400
+ assert "text blocks" in (result.error_message or "")
@pytest.mark.parametrize("location", ["us", "eu", "us-central1", "europe-west1", "global"])
@@ -150,15 +173,11 @@ async def test_get_supports_system_message():
from litellm.llms.vertex_ai.common_utils import get_supports_system_message
# fine-tuned vertex gemini models will specifiy they are in the /gemini spec format
- result = get_supports_system_message(
- model="gemini/1234567890", custom_llm_provider="vertex_ai"
- )
+ result = get_supports_system_message(model="gemini/1234567890", custom_llm_provider="vertex_ai")
assert result == True
# non-fine-tuned vertex gemini models will not specifiy they are in the /gemini spec format
- result = get_supports_system_message(
- model="random-model-name", custom_llm_provider="vertex_ai"
- )
+ result = get_supports_system_message(model="random-model-name", custom_llm_provider="vertex_ai")
assert result == False
@@ -231,13 +250,9 @@ def test_build_vertex_schema():
"properties": {
"tags": {"items": {"type": "string"}, "type": "array"},
"metadata": {"type": "object"},
- "callbacks": {
- "anyOf": [{"items": {}, "type": "array"}, {}, {"type": "null"}]
- },
+ "callbacks": {"anyOf": [{"items": {}, "type": "array"}, {}, {"type": "null"}]},
"run_name": {"type": "string"},
- "max_concurrency": {
- "anyOf": [{"type": "integer"}, {"type": "null"}]
- },
+ "max_concurrency": {"anyOf": [{"type": "integer"}, {"type": "null"}]},
"recursion_limit": {"type": "integer"},
"configurable": {"type": "object"},
"run_id": {
@@ -281,9 +296,7 @@ def test_build_vertex_schema():
]
},
"run_name": {"type": "string"},
- "max_concurrency": {
- "anyOf": [{"type": "integer", "nullable": True}]
- },
+ "max_concurrency": {"anyOf": [{"type": "integer", "nullable": True}]},
"recursion_limit": {"type": "integer"},
"configurable": {"type": "object"},
"run_id": {"anyOf": [{"type": "string", "nullable": True}]},
@@ -688,9 +701,7 @@ def test_get_vertex_url_global_region(stream, expected_endpoint_suffix):
({}, "us-east1", "us-east1"),
],
)
-def test_get_vertex_region_global_only_model(
- model_cost_entry, vertex_region, expected_region
-):
+def test_get_vertex_region_global_only_model(model_cost_entry, vertex_region, expected_region):
"""Test get_vertex_region resolves region from model_cost supported_regions"""
import litellm
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@@ -702,9 +713,7 @@ def test_get_vertex_region_global_only_model(
{"vertex_ai/test-model": model_cost_entry},
clear=False,
):
- result = vertex_base.get_vertex_region(
- vertex_region=vertex_region, model="test-model"
- )
+ result = vertex_base.get_vertex_region(vertex_region=vertex_region, model="test-model")
assert result == expected_region
@@ -822,9 +831,7 @@ def test_convert_schema_types_type_array_conversion():
assert anyof_types[1]["type"] == "number"
# 4. Other properties preserved
- assert (
- input_schema["properties"]["studio"]["description"] == "The studio ID or name"
- )
+ assert input_schema["properties"]["studio"]["description"] == "The studio ID or name"
assert input_schema["required"] == ["studio"]
@@ -991,7 +998,9 @@ def test_construct_target_url_with_version_prefix():
),
],
)
-def test_construct_target_url_versionless_project_route_gets_api_version(requested_route: str, expected_url: str) -> None:
+def test_construct_target_url_versionless_project_route_gets_api_version(
+ requested_route: str, expected_url: str
+) -> None:
from litellm.llms.vertex_ai.common_utils import construct_target_url
target_url = construct_target_url(
@@ -1124,10 +1133,7 @@ def test_fix_enum_types():
# 2. Non-string enums are removed
assert "enum" not in input_schema["properties"]["maxLength"]
assert "enum" not in input_schema["properties"]["enabled"]
- assert (
- "enum"
- not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"]
- )
+ assert "enum" not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"]
# 3. anyOf with string type keeps enum, non-string removes it
assert "enum" in input_schema["properties"]["anyOfField"]["anyOf"][0]
@@ -1181,8 +1187,6 @@ def test_get_token_url():
assert "v1beta1" not in url
assert "/v1/" in url
- pass
-
@pytest.mark.asyncio
async def test_vertex_ai_token_counter_routes_partner_models():
@@ -1460,6 +1464,49 @@ async def test_vertex_ai_token_counter_routes_gemini_models():
assert result.total_tokens == 50
+@pytest.mark.asyncio
+async def test_vertex_gemini_count_tokens_uses_request_system_over_deployment_system_instruction() -> None:
+ class VertexCountTokensPeer:
+ def __init__(self) -> None:
+ self.request_body: dict[str, JsonValue] | None = None
+
+ def __call__(self, request: httpx.Request) -> httpx.Response:
+ self.request_body = TypeAdapter(dict[str, JsonValue]).validate_json(request.content)
+ return httpx.Response(200, json={"totalTokens": 37}, request=request)
+
+ peer: Final = VertexCountTokensPeer()
+ client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(peer))
+
+ with (
+ patch.object(gemini_count_tokens_handler, "get_async_httpx_client", return_value=client),
+ patch.object(
+ vertex_count_tokens_handler.VertexAITokenCounter,
+ "_ensure_access_token_async",
+ new=AsyncMock(return_value=("fake-token", "test-project")),
+ ),
+ ):
+ result: Final = await VertexAITokenCounter().count_tokens(
+ model_to_use="gemini-2.5-flash",
+ messages=[{"role": "user", "content": "Hello"}],
+ contents=None,
+ deployment={
+ "litellm_params": {
+ "vertex_project": "test-project",
+ "vertex_location": "us-central1",
+ "system_instruction": {"parts": [{"text": "deployment-level"}]},
+ }
+ },
+ request_model="vertex_ai/gemini-2.5-flash",
+ system="You are terse.",
+ )
+
+ assert result is not None
+ assert result.error is False
+ assert result.total_tokens == 37
+ assert peer.request_body is not None
+ assert peer.request_body["systemInstruction"] == {"parts": [{"text": "You are terse."}]}
+
+
@pytest.mark.asyncio
async def test_vertex_ai_token_counter_converts_messages_to_contents_for_gemini():
"""
@@ -1474,9 +1521,7 @@ async def test_vertex_ai_token_counter_converts_messages_to_contents_for_gemini(
token_counter = VertexAITokenCounter()
- with patch(
- "litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens"
- ) as mock_acount_tokens:
+ with patch("litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens") as mock_acount_tokens:
mock_acount_tokens.return_value = {
"totalTokens": 42,
"tokenizer_used": "gemini",
@@ -1518,9 +1563,7 @@ async def test_vertex_ai_token_counter_returns_none_when_api_omits_total_tokens(
token_counter = VertexAITokenCounter()
- with patch(
- "litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens"
- ) as mock_acount_tokens:
+ with patch("litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens") as mock_acount_tokens:
mock_acount_tokens.return_value = {"tokenizer_used": "gemini"}
result = await token_counter.count_tokens(
@@ -1563,9 +1606,7 @@ async def test_vertex_ai_partner_model_detection():
# Test Minimax models
assert VertexAIPartnerModels.is_vertex_partner_model("minimaxai/minimax-m2-maas")
# Test Moonshot models
- assert VertexAIPartnerModels.is_vertex_partner_model(
- "moonshotai/kimi-k2-thinking-maas"
- )
+ assert VertexAIPartnerModels.is_vertex_partner_model("moonshotai/kimi-k2-thinking-maas")
# Test Gemini models (should NOT be detected as partner model)
assert not VertexAIPartnerModels.is_vertex_partner_model("gemini-1.5-pro")
@@ -1596,9 +1637,7 @@ def test_vertex_ai_moonshot_uses_openai_handler():
VertexAIPartnerModels,
)
- assert VertexAIPartnerModels.should_use_openai_handler(
- "moonshotai/kimi-k2-thinking-maas"
- )
+ assert VertexAIPartnerModels.should_use_openai_handler("moonshotai/kimi-k2-thinking-maas")
def test_vertex_ai_zai_uses_openai_handler():
@@ -1633,9 +1672,7 @@ def test_vertex_ai_gemma_maas_is_partner_model():
VertexAIPartnerModels,
)
- assert VertexAIPartnerModels.is_vertex_partner_model(
- "google/gemma-4-26b-a4b-it-maas"
- )
+ assert VertexAIPartnerModels.is_vertex_partner_model("google/gemma-4-26b-a4b-it-maas")
def test_vertex_ai_gemma_maas_uses_openai_handler():
@@ -1646,9 +1683,7 @@ def test_vertex_ai_gemma_maas_uses_openai_handler():
VertexAIPartnerModels,
)
- assert VertexAIPartnerModels.should_use_openai_handler(
- "google/gemma-4-26b-a4b-it-maas"
- )
+ assert VertexAIPartnerModels.should_use_openai_handler("google/gemma-4-26b-a4b-it-maas")
def test_vertex_ai_gemma_maas_routes_to_partner_models():
@@ -1838,12 +1873,8 @@ def test_pop_vertex_request_labels_prefers_explicit_labels_then_metadata():
def test_pop_vertex_request_labels_uses_litellm_metadata_when_metadata_absent():
optional: dict = {}
- litellm_params = {
- "litellm_metadata": {"requester_metadata": {"team": "from_litellm_meta"}}
- }
- assert pop_vertex_request_labels(optional, litellm_params) == {
- "team": "from_litellm_meta"
- }
+ litellm_params = {"litellm_metadata": {"requester_metadata": {"team": "from_litellm_meta"}}}
+ assert pop_vertex_request_labels(optional, litellm_params) == {"team": "from_litellm_meta"}
def test_vertex_text_embedding_request_includes_labels_from_metadata():
@@ -1853,9 +1884,7 @@ def test_vertex_text_embedding_request_includes_labels_from_metadata():
input="hi",
optional_params={},
model="text-embedding-004",
- litellm_params={
- "metadata": {"requester_metadata": {"project_id": "cost-center-1"}}
- },
+ litellm_params={"metadata": {"requester_metadata": {"project_id": "cost-center-1"}}},
)
assert req.get("labels") == {"project_id": "cost-center-1"}
diff --git a/tests/unit/proxy/google_endpoints/test_endpoints.py b/tests/unit/proxy/google_endpoints/test_endpoints.py
index 92001118e2c..17af52b5579 100644
--- a/tests/unit/proxy/google_endpoints/test_endpoints.py
+++ b/tests/unit/proxy/google_endpoints/test_endpoints.py
@@ -2,15 +2,85 @@
Test for google_endpoints/endpoints.py
"""
+from typing import Final
+
import pytest
-import sys, os
from dotenv import load_dotenv
-
-
-from litellm.proxy.google_endpoints.endpoints import google_count_tokens
-from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
+from pydantic import JsonValue
from starlette.requests import Request
+from litellm.proxy.google_endpoints.endpoints import _google_count_tokens_request, google_count_tokens
+from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
+
+
+@pytest.mark.parametrize(
+ ("system_key", "schema_key"),
+ (("systemInstruction", "parameters"), ("system_instruction", "parametersJsonSchema")),
+)
+def test_google_count_tokens_translates_system_and_function_declarations(
+ system_key: str, schema_key: str
+) -> None:
+ schema: Final = {"type": "object", "properties": {"query": {"type": "string"}}}
+ request_body: Final[dict[str, JsonValue]] = {
+ "contents": [{"role": "user", "parts": [{"text": "Look this up"}]}],
+ system_key: {"parts": [{"text": "Use the provided tool."}]},
+ "tools": [
+ {
+ "functionDeclarations": [
+ {
+ "name": "lookup",
+ "description": "Find a record",
+ schema_key: schema,
+ }
+ ]
+ }
+ ],
+ }
+ request: Final = _google_count_tokens_request("claude-sonnet", request_body)
+
+ assert request.system == [{"type": "text", "text": "Use the provided tool."}]
+ assert request.tools == [
+ {
+ "name": "lookup",
+ "description": "Find a record",
+ "input_schema": schema,
+ }
+ ]
+ assert request.messages == [{"role": "user", "content": "Look this up"}]
+
+
+@pytest.mark.parametrize(
+ ("schema_key", "expected_schema"),
+ (
+ ("parameters", {"type": "object", "properties": {"query": {"type": "string"}}}),
+ ("parametersJsonSchema", {"type": "OBJECT", "properties": {"query": {"type": "STRING"}}}),
+ ),
+)
+def test_google_count_tokens_normalizes_parameters_only(
+ schema_key: str, expected_schema: dict[str, JsonValue]
+) -> None:
+ request: Final = _google_count_tokens_request(
+ "claude-sonnet",
+ {
+ "contents": [{"role": "user", "parts": [{"text": "Look this up"}]}],
+ "tools": [
+ {
+ "functionDeclarations": [
+ {
+ "name": "lookup",
+ schema_key: {
+ "type": "OBJECT",
+ "properties": {"query": {"type": "STRING"}},
+ },
+ }
+ ]
+ }
+ ],
+ },
+ )
+
+ assert request.tools == [{"name": "lookup", "input_schema": expected_schema}]
+
load_dotenv()