mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(count_tokens): count grok-4-1, grok-4.3 and DeepSeek V4.1 tool requests the way their hosts bill them (#49074)
* fix(count_tokens): count grok-4-1, grok-4.3 and DeepSeek V4.1 tool requests like their chat templates Render grok-4-1, grok-4.3 and DeepSeek V4.1 tool schemas the way each chat template does, and add a per-deployment token_count_tool_framing next to token_count_framing for the hidden tool preamble a host bills. The Azure AI Foundry grok-4-1, xAI grok-4.3 and Fireworks DeepSeek V4.1 cost map rows carry the measured framing, overridable in model_info * fix(count_tokens): count Azure grok-4-1 Responses requests with their own framing and use the tool framing in place of the base framing * fix(count_tokens): keep operator framing on tool and Responses counts, frame bridged Responses like chat, follow base_model for tool rendering, count grok tool_choice * fix(token_counter): count Azure Responses strict default, skip false strict and built-in-only tools, per-kind operator framing * fix(token_counter): count an explicit null strict like false on Azure Responses * fix(count_tokens): follow the chat completions bridge flag and Azure's strict default on azure/ deployments Keep Azure grok-4-1 chat tool counts at or over the bill by raising its tool framing to 302 * fix(token_counter): keep strict false for non-grok templates, coerce the bridge flag, frame DeepSeek API counts * fix(proxy): frame Responses counts for the api_base router defaults send inference to * fix(proxy): keep the api_base helper's name so main's /v1/messages count caller resolves --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
tavren27
GitHub
parent
6be98d8c75
commit
a4de720144
17 changed files with 904 additions and 37 deletions
|
|
@ -186,6 +186,9 @@ pub struct ModelInfoFields {
|
|||
pub supports_image_size: Option<bool>,
|
||||
pub supported_audio_formats: Option<Value>,
|
||||
pub token_count_framing: Option<i64>,
|
||||
pub token_count_tool_framing: Option<i64>,
|
||||
pub token_count_responses_framing: Option<i64>,
|
||||
pub token_count_responses_tool_framing: Option<i64>,
|
||||
pub token_count_model: Option<Value>,
|
||||
pub token_count_tokenizer: Option<Value>,
|
||||
pub vertex_ai_audio_api: Option<Value>,
|
||||
|
|
|
|||
|
|
@ -10,12 +10,18 @@ class ChatTemplateToolSection:
|
|||
preamble_tokens: int
|
||||
implicit_system_turn_tokens: int
|
||||
render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
|
||||
counts_tool_choice: bool = False
|
||||
renders_strict_only_when_true: bool = False
|
||||
|
||||
|
||||
def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
|
||||
|
||||
|
||||
def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions)
|
||||
|
||||
|
||||
def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
return "".join(
|
||||
"\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
|
||||
|
|
@ -23,13 +29,29 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
|||
|
||||
|
||||
_TOOL_SECTIONS: Final = (
|
||||
(
|
||||
re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE),
|
||||
ChatTemplateToolSection(211, 0, _json_line_tool_schemas),
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE
|
||||
),
|
||||
ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
|
||||
ChatTemplateToolSection(222, 0, _json_line_tool_schemas),
|
||||
),
|
||||
(re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
|
||||
(
|
||||
re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
|
||||
ChatTemplateToolSection(
|
||||
0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
|
||||
),
|
||||
),
|
||||
(
|
||||
re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
|
||||
ChatTemplateToolSection(
|
||||
0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -990,7 +990,11 @@ def _count_extra(
|
|||
tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known.
|
||||
"""
|
||||
reply_primer_tokens: Final = 3
|
||||
tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice)
|
||||
tool_choice_tokens: Final = (
|
||||
_count_tool_choice(count_function, tool_choice)
|
||||
if tool_section is None or tool_section.counts_tool_choice
|
||||
else 0
|
||||
)
|
||||
return (
|
||||
reply_primer_tokens
|
||||
+ _count_tool_definitions(count_function, tools, includes_system_message, tool_section)
|
||||
|
|
@ -1029,14 +1033,29 @@ def _count_tool_choice(
|
|||
return 0
|
||||
|
||||
|
||||
def _named_function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
|
||||
return (function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name"))
|
||||
|
||||
|
||||
def sends_function_tools(tools: Sequence[object] | None) -> bool:
|
||||
return tools is not None and any(_named_function_definitions(tools))
|
||||
|
||||
|
||||
def _without_false_strict(function: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {key: value for key, value in function.items() if key != "strict" or value}
|
||||
|
||||
|
||||
def _count_chat_template_tools(
|
||||
count_function: TokenCounterFunction,
|
||||
tools: Sequence[object],
|
||||
includes_system_message: bool,
|
||||
tool_section: ChatTemplateToolSection,
|
||||
) -> int:
|
||||
functions: Final = tuple(
|
||||
function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")
|
||||
named_functions: Final = tuple(_named_function_definitions(tools))
|
||||
functions: Final = (
|
||||
tuple(map(_without_false_strict, named_functions))
|
||||
if tool_section.renders_strict_only_when_true
|
||||
else named_functions
|
||||
)
|
||||
if not functions:
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -13478,6 +13478,10 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"token_count_framing": 149,
|
||||
"token_count_tool_framing": 302,
|
||||
"token_count_responses_framing": 145,
|
||||
"token_count_responses_tool_framing": 100,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
|
|
@ -54457,6 +54461,8 @@
|
|||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"litellm_provider": "xai",
|
||||
"token_count_framing": 185,
|
||||
"token_count_tool_framing": 214,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
|
|
@ -62640,7 +62646,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 23,
|
||||
"token_count_tool_framing": 12
|
||||
},
|
||||
"deepseek/deepseek-v4-flash-vision-exp": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -62746,7 +62754,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": false,
|
||||
"token_count_framing": 76,
|
||||
"token_count_tool_framing": 75
|
||||
},
|
||||
"tencent/deepseek-v4-pro": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -64032,6 +64042,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
@ -64133,6 +64144,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
|
|||
|
|
@ -56,6 +56,22 @@ class TokenCountSettings:
|
|||
framing: int | None
|
||||
count_model: TokenCountModel | None
|
||||
tokenizer: TokenCountTokenizer | None = None
|
||||
tool_framing: int | None = None
|
||||
responses_framing: int | None = None
|
||||
responses_tool_framing: int | None = None
|
||||
|
||||
@property
|
||||
def frames_counts(self) -> bool:
|
||||
return any(
|
||||
framing is not None
|
||||
for framing in (self.framing, self.tool_framing, self.responses_framing, self.responses_tool_framing)
|
||||
)
|
||||
|
||||
def hidden_prompt_tokens(self, carries_tools: bool, responses_api: bool) -> int:
|
||||
chat: Final = self.tool_framing if carries_tools else None
|
||||
responses: Final = self.responses_tool_framing if carries_tools else self.responses_framing
|
||||
narrower: Final = responses if responses_api else chat
|
||||
return (narrower if narrower is not None else self.framing) or 0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -105,6 +121,15 @@ def _framing(value: object) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def _first_framing(source: Mapping[str, object], keys: tuple[str, ...]) -> int | None:
|
||||
return next((framing for framing in map(_framing, map(source.get, keys)) if framing is not None), None)
|
||||
|
||||
|
||||
def _configured_framing(configured: Mapping[str, object], default: Mapping[str, object], *keys: str) -> int | None:
|
||||
operator_sets_one: Final = _first_framing(configured, (*keys, "token_count_framing")) is not None
|
||||
return _first_framing(configured if operator_sets_one else default, keys)
|
||||
|
||||
|
||||
def _non_blank(value: object) -> str | None:
|
||||
return value if isinstance(value, str) and value.strip() else None
|
||||
|
||||
|
|
@ -175,13 +200,21 @@ def _tokenizer(value: object) -> TokenCountTokenizer | None:
|
|||
def resolve_token_count_settings(
|
||||
configured: Mapping[str, object], default: Mapping[str, object], deployment_region: Callable[[], str]
|
||||
) -> TokenCountSettings:
|
||||
framing: Final = _framing(configured.get("token_count_framing"))
|
||||
return TokenCountSettings(
|
||||
framing=framing if framing is not None else _framing(default.get("token_count_framing")),
|
||||
framing=_configured_framing(configured, default, "token_count_framing"),
|
||||
count_model=_resolved_count_model(
|
||||
configured.get("token_count_model"), default.get("token_count_model"), deployment_region
|
||||
),
|
||||
tokenizer=_tokenizer(default.get("token_count_tokenizer")),
|
||||
tool_framing=_configured_framing(configured, default, "token_count_tool_framing"),
|
||||
responses_framing=_configured_framing(configured, default, "token_count_responses_framing"),
|
||||
responses_tool_framing=_configured_framing(
|
||||
configured,
|
||||
default,
|
||||
"token_count_responses_tool_framing",
|
||||
"token_count_responses_framing",
|
||||
"token_count_tool_framing",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
initialize_sentry,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
|
||||
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.routing_groups import parse_routing_groups
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
|
@ -115,6 +116,7 @@ from litellm.litellm_core_utils.token_counter import (
|
|||
UnsupportedContentTypeError,
|
||||
deployment_openai_model,
|
||||
offload_token_count,
|
||||
sends_function_tools,
|
||||
)
|
||||
from litellm.llms.bedrock.count_tokens.kimi_k3 import (
|
||||
HUGGING_FACE_SOURCE,
|
||||
|
|
@ -1007,6 +1009,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router
|
|||
from litellm.repositories.base_repository import SupportsModelDump
|
||||
from litellm.repositories.credentials_repository import CredentialsRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.responses.main import provider_stores_the_conversation
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.router import (
|
||||
AssistantsTypedDict,
|
||||
|
|
@ -15037,7 +15040,7 @@ async def _provider_token_count(
|
|||
framing_can_stand_in: Final = (count_model is None or not count_model.configured) and (
|
||||
failure.cannot_count_model or (pinned_count_model is not None and failure.denied_in_region)
|
||||
)
|
||||
if not (framing_can_stand_in and settings.framing is not None and local_estimate_allowed):
|
||||
if not (framing_can_stand_in and settings.frames_counts and local_estimate_allowed):
|
||||
raise failure.to_proxy_exception()
|
||||
if pinned_count_model is not None:
|
||||
verbose_proxy_logger.warning(
|
||||
|
|
@ -15237,6 +15240,7 @@ async def _tokenizer_or_provider_error(
|
|||
|
||||
async def _local_token_estimate(
|
||||
model: str,
|
||||
template_model: str,
|
||||
prompt: str | None,
|
||||
messages: Sequence[AllMessageValues] | None,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
|
|
@ -15249,7 +15253,7 @@ async def _local_token_estimate(
|
|||
else await _tokenizer_or_provider_error(model, custom_tokenizer, provider_failure)
|
||||
)
|
||||
count: Final = offload_token_count(litellm.token_counter)(
|
||||
model=model,
|
||||
model=template_model,
|
||||
text=prompt,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
|
|
@ -15315,10 +15319,15 @@ def _deployment_aws_region(deployment: _TokenCountDeployment) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _deployment_base_model(deployment: _TokenCountDeployment) -> str | None:
|
||||
return _non_empty_str(deployment.get("model_info", {}).get("base_model")) or _non_empty_str(
|
||||
deployment.get("litellm_params", {}).get("base_model")
|
||||
)
|
||||
|
||||
|
||||
def _token_count_settings_model(deployment: _TokenCountDeployment, request_model: str) -> tuple[str, str | None]:
|
||||
model_info: Final = deployment.get("model_info", {})
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
base_model: Final = _non_empty_str(model_info.get("base_model")) or _non_empty_str(params.get("base_model"))
|
||||
base_model: Final = _deployment_base_model(deployment)
|
||||
if base_model is not None:
|
||||
return base_model, None
|
||||
model: Final = _non_empty_str(params.get("model"))
|
||||
|
|
@ -15470,12 +15479,15 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) ->
|
|||
return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {}
|
||||
|
||||
|
||||
def _inference_params(
|
||||
litellm_params: Mapping[str, object], responses_request: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request}
|
||||
|
||||
|
||||
def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy:
|
||||
litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {}
|
||||
router_defaults: Final[Mapping[str, object]] = (
|
||||
{} if llm_router is None else _router_default_litellm_params(llm_router)
|
||||
)
|
||||
return drop_policy({**litellm_params, **router_defaults, **responses_request})
|
||||
return drop_policy(_inference_params(litellm_params, responses_request))
|
||||
|
||||
|
||||
def _native_responses_count(
|
||||
|
|
@ -15498,8 +15510,54 @@ def _native_responses_count(
|
|||
)
|
||||
|
||||
|
||||
def _deployment_api_base(deployment: _TokenCountDeployment) -> str | None:
|
||||
return _non_empty_str(deployment.get("litellm_params", {}).get("api_base"))
|
||||
def _deployment_api_base(
|
||||
deployment: _TokenCountDeployment, responses_request: Mapping[str, object] | None = None
|
||||
) -> str | None:
|
||||
return _non_empty_str(
|
||||
_inference_params(deployment.get("litellm_params", {}), responses_request or {}).get("api_base")
|
||||
)
|
||||
|
||||
|
||||
def _deployment_provider(
|
||||
deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object] | None
|
||||
) -> str | None:
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
try:
|
||||
return litellm.get_llm_provider(
|
||||
model=_non_empty_str(params.get("model")) or request_model,
|
||||
custom_llm_provider=_non_empty_str(params.get("custom_llm_provider")),
|
||||
api_base=_deployment_api_base(deployment, responses_request),
|
||||
)[1]
|
||||
except litellm.BadRequestError:
|
||||
return None
|
||||
|
||||
|
||||
def _serves_native_responses(
|
||||
deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object]
|
||||
) -> bool:
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
inference: Final = _inference_params(params, responses_request)
|
||||
try:
|
||||
return provider_stores_the_conversation(
|
||||
_non_empty_str(params.get("model")) or request_model,
|
||||
_non_empty_str(params.get("custom_llm_provider")),
|
||||
{
|
||||
"use_chat_completions_api": bool(inference.get("use_chat_completions_api")),
|
||||
"model_info": dict(deployment.get("model_info", {})),
|
||||
"api_base": _non_empty_str(inference.get("api_base")),
|
||||
},
|
||||
)
|
||||
except litellm.BadRequestError:
|
||||
return False
|
||||
|
||||
|
||||
_AZURE_PROVIDERS: Final = frozenset((litellm.LlmProviders.AZURE.value, litellm.LlmProviders.AZURE_AI.value))
|
||||
|
||||
|
||||
def _with_strict_by_default(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
{**tool, "strict": True} if tool.get("type") == "function" and "strict" not in tool else tool for tool in tools
|
||||
)
|
||||
|
||||
|
||||
def _system_message(system: object) -> ChatCompletionSystemMessage | None:
|
||||
|
|
@ -15704,7 +15762,7 @@ async def _count_request_tokens(
|
|||
else await chat_count_request(
|
||||
model_to_use,
|
||||
custom_llm_provider,
|
||||
_deployment_api_base(deployment_view),
|
||||
_deployment_api_base(deployment_view, routed_request),
|
||||
routed_request,
|
||||
_count_drop_policy(routed_deployment, routed_request),
|
||||
)
|
||||
|
|
@ -15802,10 +15860,31 @@ async def _count_request_tokens(
|
|||
counted_messages: Final = (
|
||||
typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages)
|
||||
)
|
||||
counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
|
||||
native_responses: Final = responses_request is not None and _serves_native_responses(
|
||||
deployment_view, request.model, responses_request
|
||||
)
|
||||
sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
|
||||
list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None
|
||||
)
|
||||
framing: Final = settings.framing if counted_messages is not None and settings.framing is not None else 0
|
||||
openai_model: Final = deployment_openai_model(deployment_view)
|
||||
template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use
|
||||
tool_section: Final = chat_template_tool_section(template_model)
|
||||
counted_tools: Final = (
|
||||
cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added
|
||||
if sent_tools is not None
|
||||
and native_responses
|
||||
and _deployment_provider(deployment_view, request.model, responses_request) in _AZURE_PROVIDERS
|
||||
and tool_section is not None
|
||||
and tool_section.renders_strict_only_when_true
|
||||
else sent_tools
|
||||
)
|
||||
framing: Final = (
|
||||
0
|
||||
if counted_messages is None
|
||||
else settings.hidden_prompt_tokens(
|
||||
carries_tools=sends_function_tools(counted_tools), responses_api=native_responses
|
||||
)
|
||||
)
|
||||
kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None
|
||||
exact: Final = (
|
||||
await _kimi_k3_count(
|
||||
|
|
@ -15829,7 +15908,8 @@ async def _count_request_tokens(
|
|||
count_source="local_estimate",
|
||||
)
|
||||
estimate: Final = await _local_token_estimate(
|
||||
model=deployment_openai_model(deployment_view) or model_to_use,
|
||||
model=openai_model or model_to_use,
|
||||
template_model=template_model,
|
||||
prompt=prompt if counted_messages is None else None,
|
||||
messages=counted_messages,
|
||||
tools=counted_tools,
|
||||
|
|
|
|||
|
|
@ -246,6 +246,9 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
bedrock_converse_supports_strict_tools: bool | None
|
||||
supports_regex_lookaround: ReadOnly[bool | None]
|
||||
token_count_framing: ReadOnly[int | None]
|
||||
token_count_tool_framing: ReadOnly[int | None]
|
||||
token_count_responses_framing: ReadOnly[int | None]
|
||||
token_count_responses_tool_framing: ReadOnly[int | None]
|
||||
token_count_model: ReadOnly[TokenCountModelInfo | None]
|
||||
token_count_tokenizer: ReadOnly[TokenCountTokenizer | None]
|
||||
|
||||
|
|
@ -4236,7 +4239,15 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})
|
||||
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"token_count_framing",
|
||||
"token_count_tool_framing",
|
||||
"token_count_responses_framing",
|
||||
"token_count_responses_tool_framing",
|
||||
"token_count_model",
|
||||
}
|
||||
)
|
||||
|
||||
SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
|
||||
frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)
|
||||
|
|
|
|||
|
|
@ -7398,6 +7398,9 @@ def get_model_info_helper(
|
|||
supports_image_size=_model_info.get("supports_image_size", None),
|
||||
supported_audio_formats=_model_info.get("supported_audio_formats", None),
|
||||
token_count_framing=_model_info.get("token_count_framing", None),
|
||||
token_count_tool_framing=_model_info.get("token_count_tool_framing", None),
|
||||
token_count_responses_framing=_model_info.get("token_count_responses_framing", None),
|
||||
token_count_responses_tool_framing=_model_info.get("token_count_responses_tool_framing", None),
|
||||
token_count_model=_model_info.get("token_count_model", None),
|
||||
token_count_tokenizer=_model_info.get("token_count_tokenizer", None),
|
||||
vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None),
|
||||
|
|
|
|||
|
|
@ -13478,6 +13478,10 @@
|
|||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"token_count_framing": 149,
|
||||
"token_count_tool_framing": 302,
|
||||
"token_count_responses_framing": 145,
|
||||
"token_count_responses_tool_framing": 100,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
|
|
@ -54457,6 +54461,8 @@
|
|||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"litellm_provider": "xai",
|
||||
"token_count_framing": 185,
|
||||
"token_count_tool_framing": 214,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
|
|
@ -62640,7 +62646,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 23,
|
||||
"token_count_tool_framing": 12
|
||||
},
|
||||
"deepseek/deepseek-v4-flash-vision-exp": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -62746,7 +62754,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": false,
|
||||
"token_count_framing": 76,
|
||||
"token_count_tool_framing": 75
|
||||
},
|
||||
"tencent/deepseek-v4-pro": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -64032,6 +64042,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
@ -64133,6 +64144,7 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_priority": 3.75e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"token_count_framing": 23,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"mode": "chat",
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ _API_KEY: Final = "scripted-provider-key"
|
|||
_DEEPSEEK_V4: Final = "azure_ai/DeepSeek-V4-Flash-0731"
|
||||
_DEEPSEEK_R1: Final = "azure_ai/DeepSeek-R1-0528"
|
||||
_QWEN3_5: Final = "databricks/databricks-qwen35-122b-a10b"
|
||||
_GROK: Final = "azure_ai/grok-4-1"
|
||||
_GROK_3: Final = "azure_ai/grok-3"
|
||||
_DEEPSEEK_V4_TOOL_PREAMBLE: Final = 222
|
||||
_QWEN3_5_TOOL_PREAMBLE: Final = 196
|
||||
_QWEN3_5_OPENED_SYSTEM_TURN: Final = 5
|
||||
|
|
@ -250,7 +250,7 @@ def test_messages_count_tokens_skips_web_search_and_counts_unmapped_tool_keys_fo
|
|||
|
||||
def test_messages_count_tokens_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None:
|
||||
with wire_server(_completion) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, "count", _GROK, wire.url)
|
||||
model: Final = _deployment(scenario, "count", _GROK_3, wire.url)
|
||||
without_tools: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ()))
|
||||
plain: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,)))
|
||||
deferred: Final = _count(
|
||||
|
|
@ -428,7 +428,7 @@ def test_project_itpm_reserves_deepseek_v4_tools_the_way_its_chat_template_rende
|
|||
def test_project_itpm_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None:
|
||||
chat: Final = _ENDPOINTS["chat-completions"]
|
||||
with wire_server(_completion) as wire, gateway.scenario() as scenario:
|
||||
group: Final = _deployment(scenario, "grok-4-1", _GROK, wire.url)
|
||||
group: Final = _deployment(scenario, "grok-3", _GROK_3, wire.url)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
chat.path,
|
||||
|
|
@ -504,8 +504,8 @@ def _agent_body(group: str, marker: str, file_body: str, trigger: int) -> dict[s
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "files_kept"),
|
||||
((_DEEPSEEK_V4, (False, False, True)), (_GROK, (True, True, True))),
|
||||
ids=("deepseek-v4", "grok-4-1"),
|
||||
((_DEEPSEEK_V4, (False, False, True)), (_GROK_3, (True, True, True))),
|
||||
ids=("deepseek-v4", "grok-3"),
|
||||
)
|
||||
def test_messages_context_management_trigger_counts_tools_the_way_the_models_chat_template_renders_them(
|
||||
gateway: Gateway, model: str, files_kept: tuple[bool, ...]
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ _BARE_FUNCTION: Final = {"name": "g", "parameters": {}}
|
|||
|
||||
|
||||
# Tool schemas as rendered by render_tools in encoding/encoding_dsv4.py (deepseek-ai/DeepSeek-V4-Flash) and by
|
||||
# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17
|
||||
# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17, and by chat_template.jinja
|
||||
# (deepseek-ai/DeepSeek-V4.1-Flash) read on 2026-10-29. Grok's template is closed, so its two renderings are the ones
|
||||
# whose token counts left a constant preamble across seven tool shapes billed by Azure AI Foundry
|
||||
# grok-4-1-fast-reasoning and xAI grok-4.3 on 2026-10-29
|
||||
@pytest.mark.parametrize(
|
||||
("model", "rendered"),
|
||||
[
|
||||
|
|
@ -17,6 +20,18 @@ _BARE_FUNCTION: Final = {"name": "g", "parameters": {}}
|
|||
"DeepSeek-V4-Flash-0731",
|
||||
'{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}',
|
||||
),
|
||||
(
|
||||
"DeepSeek-V4.1-Flash",
|
||||
'{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}',
|
||||
),
|
||||
(
|
||||
"grok-4-1-fast-reasoning",
|
||||
'{"name":"f","description":"é","parameters":{}}\n{"name":"g","parameters":{}}',
|
||||
),
|
||||
(
|
||||
"grok-4.3",
|
||||
'{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}',
|
||||
),
|
||||
(
|
||||
"databricks-qwen35-122b-a10b",
|
||||
'\n{"type": "function", "function": {"name": "f", "description": "é", "parameters": {}}}'
|
||||
|
|
|
|||
|
|
@ -638,8 +638,14 @@ def test_token_counter_tools_match_billed_prompt_for_json_tool_chat_templates(
|
|||
("deepseek/deepseek-chat:free", "DeepSeek-V4-Flash-0731"),
|
||||
("deepseek-reasoner", "DeepSeek-V4-Flash-0731"),
|
||||
("deepseek/deepseek-flash", "DeepSeek-V4-Flash-0731"),
|
||||
("DeepSeek-V4.1-Flash", "deepseek-v3.2"),
|
||||
("accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v3.2"),
|
||||
("DeepSeek-V4.1-Flash", "accounts/fireworks/models/deepseek-v4p1-flash"),
|
||||
("databricks-deepseek-v4-1-flash", "accounts/fireworks/models/deepseek-v4p1-flash"),
|
||||
("deepseek-ai/deepseek-v4.1-flash-maas", "accounts/fireworks/models/deepseek-v4p1-flash"),
|
||||
("deepseek-v4p1-flash-1017", "accounts/fireworks/models/deepseek-v4p1-flash"),
|
||||
("grok-4.1-fast-non-reasoning", "grok-4-1-fast-reasoning"),
|
||||
("xai.grok-4.1-fast", "grok-4-1-fast-reasoning"),
|
||||
("grok-4.3-latest", "grok-4.3"),
|
||||
("xai.grok-4-3", "grok-4.3"),
|
||||
("Qwen/Qwen3.5-122B-A10B", "databricks-qwen35-122b-a10b"),
|
||||
("qwen3-235b-a22b", "mistral-large-latest"),
|
||||
],
|
||||
|
|
@ -704,6 +710,48 @@ def test_token_counter_tool_choice_adds_nothing_for_deepseek_v4(tool_choice: obj
|
|||
) == token_counter_new(model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools)
|
||||
|
||||
|
||||
# A request with a one-function tool billed 283 prompt tokens on xAI grok-4.3 without tool_choice and 297 with a
|
||||
# named one, and 357 and 358 on Azure AI Foundry grok-4-1-fast-reasoning, on 2026-10-29
|
||||
@pytest.mark.parametrize("model", ["grok-4.3", "grok-4-1-fast-reasoning"])
|
||||
def test_token_counter_counts_a_named_tool_choice_for_grok(model: str):
|
||||
tools: Final = [_openai_tool(_WEATHER_FUNCTION)]
|
||||
named: Final = {"type": "function", "function": {"name": "get_weather"}}
|
||||
|
||||
assert token_counter_new(
|
||||
model=model, messages=list(_USER_MESSAGES), tools=tools, tool_choice=named
|
||||
) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools)
|
||||
|
||||
|
||||
def _counted_with_strict(model: str, **strict: bool) -> int:
|
||||
return token_counter_new(
|
||||
model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})]
|
||||
)
|
||||
|
||||
|
||||
# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI
|
||||
# Foundry grok-4-1-fast-reasoning chat and 283 through xAI grok-4.3
|
||||
@pytest.mark.parametrize("model", ["grok-4-1-fast-reasoning", "grok-4.3"])
|
||||
def test_token_counter_renders_a_false_strict_like_an_unset_one_for_grok(model: str):
|
||||
assert (
|
||||
_counted_with_strict(model, strict=False)
|
||||
== _counted_with_strict(model)
|
||||
< _counted_with_strict(model, strict=True)
|
||||
)
|
||||
|
||||
|
||||
# A one-function tool billed 5 prompt tokens more with strict false than unset, the same as with strict true, on
|
||||
# 2026-10-30: 310 against 305 through Baseten DeepSeek-V4.1-Flash and 309 against 304 through Together
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v4-flash", "qwen3.5-plus"]
|
||||
)
|
||||
def test_token_counter_renders_a_false_strict_like_a_true_one_for_other_chat_templates(model: str):
|
||||
assert (
|
||||
_counted_with_strict(model, strict=False)
|
||||
== _counted_with_strict(model, strict=True)
|
||||
> _counted_with_strict(model)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "expected_tool_choice"),
|
||||
[
|
||||
|
|
@ -759,9 +807,17 @@ def test_token_counter_counts_anthropic_tools_as_sent_to_chat_template_families(
|
|||
|
||||
# Qwen3.5's chat_template.jinja opens a system turn for the tool schemas when the request has none, 5 tokens that a
|
||||
# request with a system message already pays, while DeepSeek V4's encoding_dsv4.py appends them to the first turn
|
||||
# either way (both read on Hugging Face on 2026-10-17)
|
||||
# either way (both read on Hugging Face on 2026-10-17). A system message cost the same with and without a tool on
|
||||
# Fireworks deepseek-v4p1-flash, Azure AI Foundry grok-4-1-fast-reasoning and xAI grok-4.3 on 2026-10-29
|
||||
@pytest.mark.parametrize(
|
||||
("model", "implicit_system_turn_tokens"), [("DeepSeek-V4-Flash-0731", 0), ("databricks-qwen35-122b-a10b", 5)]
|
||||
("model", "implicit_system_turn_tokens"),
|
||||
[
|
||||
("DeepSeek-V4-Flash-0731", 0),
|
||||
("databricks-qwen35-122b-a10b", 5),
|
||||
("accounts/fireworks/models/deepseek-v4p1-flash", 0),
|
||||
("grok-4-1-fast-reasoning", 0),
|
||||
("grok-4.3", 0),
|
||||
],
|
||||
)
|
||||
def test_token_counter_counts_the_tools_system_turn_only_without_a_system_message(
|
||||
model: str, implicit_system_turn_tokens: int
|
||||
|
|
|
|||
|
|
@ -131,6 +131,107 @@ def test_shipped_australian_count_region_stays_with_an_australian_deployment() -
|
|||
assert settings.count_model == TokenCountModel(model="au.anthropic.claude-opus-5-5", region="ap-southeast-4")
|
||||
|
||||
|
||||
_ABSENT: Final = object()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key", "field"),
|
||||
[
|
||||
("token_count_tool_framing", "tool_framing"),
|
||||
("token_count_responses_framing", "responses_framing"),
|
||||
("token_count_responses_tool_framing", "responses_tool_framing"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "default", "expected"),
|
||||
[
|
||||
(_ABSENT, 151, 151),
|
||||
(40, 151, 40),
|
||||
(0, 151, 0),
|
||||
(40.0, _ABSENT, 40),
|
||||
(-1, 151, 151),
|
||||
(True, 151, 151),
|
||||
("40", _ABSENT, None),
|
||||
],
|
||||
)
|
||||
def test_each_request_kind_framing_reads_its_value_like_framing(
|
||||
key: str, field: str, configured: object, default: object, expected: int | None
|
||||
) -> None:
|
||||
settings: Final = resolve_token_count_settings(
|
||||
configured={} if configured is _ABSENT else {key: configured},
|
||||
default={} if default is _ABSENT else {key: default},
|
||||
deployment_region=_in_us_west_2,
|
||||
)
|
||||
assert getattr(settings, field) == expected
|
||||
|
||||
|
||||
def _hidden_prompts(settings: TokenCountSettings) -> tuple[int, int, int, int]:
|
||||
return (
|
||||
settings.hidden_prompt_tokens(carries_tools=False, responses_api=False),
|
||||
settings.hidden_prompt_tokens(carries_tools=True, responses_api=False),
|
||||
settings.hidden_prompt_tokens(carries_tools=False, responses_api=True),
|
||||
settings.hidden_prompt_tokens(carries_tools=True, responses_api=True),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("default", "expected"),
|
||||
[
|
||||
({}, (0, 0, 0, 0)),
|
||||
({"token_count_framing": 23}, (23, 23, 23, 23)),
|
||||
({"token_count_framing": 185, "token_count_tool_framing": 214}, (185, 214, 185, 214)),
|
||||
(
|
||||
{
|
||||
"token_count_framing": 149,
|
||||
"token_count_tool_framing": 300,
|
||||
"token_count_responses_framing": 145,
|
||||
"token_count_responses_tool_framing": 104,
|
||||
},
|
||||
(149, 300, 145, 104),
|
||||
),
|
||||
({"token_count_framing": 149, "token_count_tool_framing": 0}, (149, 0, 149, 0)),
|
||||
({"token_count_framing": 149, "token_count_responses_tool_framing": 0}, (149, 149, 149, 0)),
|
||||
({"token_count_framing": 149, "token_count_responses_framing": 145}, (149, 149, 145, 145)),
|
||||
],
|
||||
)
|
||||
def test_hidden_prompt_is_the_most_specific_framing_set_for_the_request_api_and_tools(
|
||||
default: dict[str, object], expected: tuple[int, int, int, int]
|
||||
) -> None:
|
||||
settings: Final = resolve_token_count_settings(configured={}, default=default, deployment_region=_in_us_west_2)
|
||||
assert _hidden_prompts(settings) == expected
|
||||
|
||||
|
||||
_PER_API_COST_MAP: Final = {
|
||||
"token_count_framing": 149,
|
||||
"token_count_tool_framing": 300,
|
||||
"token_count_responses_framing": 145,
|
||||
"token_count_responses_tool_framing": 104,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
({}, (149, 300, 145, 104)),
|
||||
({"token_count_framing": 500}, (500, 500, 500, 500)),
|
||||
({"token_count_framing": 0}, (0, 0, 0, 0)),
|
||||
({"token_count_tool_framing": 250}, (149, 250, 145, 250)),
|
||||
({"token_count_framing": 500, "token_count_tool_framing": 250}, (500, 250, 500, 250)),
|
||||
({"token_count_responses_framing": 120}, (149, 300, 120, 120)),
|
||||
({"token_count_tool_framing": 250, "token_count_responses_framing": 120}, (149, 250, 120, 120)),
|
||||
({"token_count_responses_tool_framing": 90}, (149, 300, 145, 90)),
|
||||
({"token_count_framing": -1}, (149, 300, 145, 104)),
|
||||
],
|
||||
)
|
||||
def test_an_operator_framing_applies_to_every_request_kind_the_cost_map_splits_more_finely(
|
||||
configured: dict[str, object], expected: tuple[int, int, int, int]
|
||||
) -> None:
|
||||
settings: Final = resolve_token_count_settings(
|
||||
configured=configured, default=_PER_API_COST_MAP, deployment_region=_in_us_west_2
|
||||
)
|
||||
assert _hidden_prompts(settings) == expected
|
||||
|
||||
|
||||
def test_no_setting_anywhere_means_the_model_counts_itself_without_framing() -> None:
|
||||
assert resolve_token_count_settings(
|
||||
configured={}, default={}, deployment_region=_in_us_west_2
|
||||
|
|
|
|||
|
|
@ -4896,6 +4896,9 @@ class TestModelInfoCostMapEchoFilter:
|
|||
[
|
||||
("bedrock/us.anthropic.claude-sonnet-5-5", "token_count_model", {}),
|
||||
("bedrock/us.moonshotai.kimi-k3", "token_count_framing", 0),
|
||||
("azure_ai/grok-4-1-fast-reasoning", "token_count_tool_framing", 0),
|
||||
("azure_ai/grok-4-1-fast-reasoning", "token_count_responses_framing", 0),
|
||||
("azure_ai/grok-4-1-fast-reasoning", "token_count_responses_tool_framing", 0),
|
||||
],
|
||||
)
|
||||
def test_token_count_setting_is_stored_only_when_the_operator_changed_it(
|
||||
|
|
|
|||
|
|
@ -1925,6 +1925,23 @@ async def test_uncountable_model_without_framing_raises():
|
|||
assert raised.value.code == "404"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"settings",
|
||||
[
|
||||
TokenCountSettings(framing=None, count_model=None, tool_framing=300),
|
||||
TokenCountSettings(framing=None, count_model=None, responses_framing=145),
|
||||
TokenCountSettings(framing=None, count_model=None, responses_tool_framing=100),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_uncountable_model_with_only_a_request_kind_framing_falls_through_to_the_local_estimate(
|
||||
settings: TokenCountSettings,
|
||||
):
|
||||
counter: Final = _ScriptedCounter({"us.anthropic.claude-sonnet-5-5": _provider_reply("sonnet", status_code=404)})
|
||||
|
||||
assert isinstance(await _ladder(counter, settings), ProviderCountFailure)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("count_model", "counted_as"),
|
||||
[
|
||||
|
|
@ -2770,6 +2787,480 @@ async def test_framing_comes_from_the_row_the_deployment_provider_or_base_model_
|
|||
assert aliased.total_tokens == litellm.token_counter(model="gpt-5.4-mini", messages=messages) + framing
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_tool_request_counts_its_framing_in_place_of_the_plain_one_per_api(monkeypatch: pytest.MonkeyPatch):
|
||||
def deployment(name: str, model_info: dict[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": name,
|
||||
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": f"sk-unused-{name}"},
|
||||
"model_info": model_info,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
deployment("plain", {}),
|
||||
deployment("chat", {"token_count_framing": 7, "token_count_tool_framing": 50}),
|
||||
deployment(
|
||||
"per-api",
|
||||
{
|
||||
"token_count_framing": 7,
|
||||
"token_count_tool_framing": 50,
|
||||
"token_count_responses_framing": 5,
|
||||
"token_count_responses_tool_framing": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
),
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "What is the weather in Paris?"}]
|
||||
tools: Final = [_KIMI_WEATHER]
|
||||
|
||||
async def count(model: str, request_tools: list[dict[str, object]] | None, responses: bool) -> int:
|
||||
response: Final = await count_request_tokens(
|
||||
request=TokenCountRequest(model=model, messages=messages, tools=request_tools),
|
||||
responses_request={"model": model, "input": "What is the weather in Paris?"} if responses else None,
|
||||
)
|
||||
return response.total_tokens
|
||||
|
||||
plain_with_tools: Final = await count("plain", tools, responses=False)
|
||||
plain_without_tools: Final = await count("plain", None, responses=False)
|
||||
|
||||
assert await count("plain", tools, responses=True) == plain_with_tools
|
||||
assert await count("chat", tools, responses=False) == plain_with_tools + 50
|
||||
assert await count("chat", None, responses=False) == plain_without_tools + 7
|
||||
assert await count("chat", [], responses=False) == plain_without_tools + 7
|
||||
assert await count("chat", tools, responses=True) == plain_with_tools + 50
|
||||
assert await count("chat", None, responses=True) == plain_without_tools + 7
|
||||
assert await count("per-api", tools, responses=False) == plain_with_tools + 50
|
||||
assert await count("per-api", tools, responses=True) == plain_with_tools + 2
|
||||
assert await count("per-api", None, responses=True) == plain_without_tools + 5
|
||||
|
||||
|
||||
_GET_WEATHER: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a city",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string", "description": "City name"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
}
|
||||
_THREE_TOOLS: Final = (
|
||||
_GET_WEATHER,
|
||||
{
|
||||
"name": "get_time",
|
||||
"description": "Get the local time in a timezone",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"timezone": {"type": "string", "description": "IANA timezone"}},
|
||||
"required": ["timezone"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "search_docs",
|
||||
"description": "Search internal documents and return the top matches",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}, "limit": {"type": "integer", "description": "Max results"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
)
|
||||
_FORM_FIELDS: Final = tuple(f"field_{index}" for index in range(10))
|
||||
_FILL_FORM: Final = {
|
||||
"name": "fill_form",
|
||||
"description": "Fill a form",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
field: {"type": "string", "description": f"Value for field number {index}"}
|
||||
for index, field in enumerate(_FORM_FIELDS)
|
||||
},
|
||||
"required": list(_FORM_FIELDS),
|
||||
},
|
||||
}
|
||||
_WEATHER_SYSTEM: Final = "You are a helpful assistant that answers weather questions briefly."
|
||||
_AZURE_GROK_4_1: Final = "azure_ai/grok-4-1-fast-reasoning"
|
||||
_FIREWORKS_DEEPSEEK_V4_1: Final = "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash"
|
||||
_XAI_GROK_4_3: Final = "xai/grok-4.3"
|
||||
_NATIVE_RESPONSES_BASE: Final = {
|
||||
_AZURE_GROK_4_1: "https://unused.services.ai.azure.com/models",
|
||||
_FIREWORKS_DEEPSEEK_V4_1: "https://unused.invalid",
|
||||
_XAI_GROK_4_3: "https://unused.invalid",
|
||||
}
|
||||
|
||||
|
||||
def _responses_function(tool: Mapping[str, object], **strict: bool | None) -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"name": tool["name"],
|
||||
"description": tool["description"],
|
||||
"parameters": tool["input_schema"],
|
||||
**strict,
|
||||
}
|
||||
|
||||
|
||||
def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]:
|
||||
return tuple({**_GET_WEATHER, "name": f"get_weather_{index}"} for index in range(count))
|
||||
|
||||
|
||||
# usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn
|
||||
# "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and
|
||||
# xAI grok-4.3 on 2026-10-29, and usage.prompt_tokens that POST /v1/chat/completions billed through DeepSeek's API on
|
||||
# 2026-10-30
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "system", "tools", "billed_prompt_tokens"),
|
||||
[
|
||||
(_AZURE_GROK_4_1, None, (), 163),
|
||||
(_AZURE_GROK_4_1, None, (_GET_WEATHER,), 357),
|
||||
(_AZURE_GROK_4_1, None, _THREE_TOOLS, 445),
|
||||
(_AZURE_GROK_4_1, None, (_FILL_FORM,), 550),
|
||||
(_AZURE_GROK_4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 368),
|
||||
(_FIREWORKS_DEEPSEEK_V4_1, None, (), 37),
|
||||
(_FIREWORKS_DEEPSEEK_V4_1, None, (_GET_WEATHER,), 304),
|
||||
(_FIREWORKS_DEEPSEEK_V4_1, None, _THREE_TOOLS, 430),
|
||||
(_FIREWORKS_DEEPSEEK_V4_1, None, (_FILL_FORM,), 550),
|
||||
(_FIREWORKS_DEEPSEEK_V4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 315),
|
||||
(_XAI_GROK_4_3, None, (), 199),
|
||||
(_XAI_GROK_4_3, None, (_GET_WEATHER,), 283),
|
||||
(_XAI_GROK_4_3, None, _THREE_TOOLS, 402),
|
||||
(_XAI_GROK_4_3, None, (_FILL_FORM,), 530),
|
||||
(_XAI_GROK_4_3, _WEATHER_SYSTEM, (_GET_WEATHER,), 297),
|
||||
("deepseek/deepseek-v4-flash", None, (), 37),
|
||||
("deepseek/deepseek-v4-flash", None, (_GET_WEATHER,), 304),
|
||||
("deepseek/deepseek-v4-flash", None, _THREE_TOOLS, 430),
|
||||
("deepseek/deepseek-v4-flash", _WEATHER_SYSTEM, (_GET_WEATHER,), 315),
|
||||
("deepseek/deepseek-v4-pro", None, (), 90),
|
||||
("deepseek/deepseek-v4-pro", None, (_GET_WEATHER,), 367),
|
||||
("deepseek/deepseek-v4-pro", None, _THREE_TOOLS, 493),
|
||||
("deepseek/deepseek-v4-pro", _WEATHER_SYSTEM, (_GET_WEATHER,), 378),
|
||||
],
|
||||
)
|
||||
async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
system: str | None,
|
||||
tools: tuple[dict[str, object], ...],
|
||||
billed_prompt_tokens: int,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "billed",
|
||||
"litellm_params": {"model": model, "api_key": "unused", "api_base": "https://unused.invalid"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
counted: Final = await token_counter(
|
||||
request=TokenCountRequest(
|
||||
model="billed",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=list(tools) or None,
|
||||
system=system,
|
||||
)
|
||||
)
|
||||
|
||||
assert counted.count_source == "local_estimate"
|
||||
assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05
|
||||
|
||||
|
||||
# usage.input_tokens that POST /v1/responses billed for these requests with the input "What is the weather in Paris?"
|
||||
# through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and xAI grok-4.3 on 2026-10-29
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "system", "tools", "billed_prompt_tokens"),
|
||||
[
|
||||
(_AZURE_GROK_4_1, None, (), 159),
|
||||
(_AZURE_GROK_4_1, None, (_GET_WEATHER,), 159),
|
||||
(_AZURE_GROK_4_1, None, _THREE_TOOLS, 257),
|
||||
(_AZURE_GROK_4_1, None, (_FILL_FORM,), 352),
|
||||
(_AZURE_GROK_4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 170),
|
||||
(_FIREWORKS_DEEPSEEK_V4_1, None, (_GET_WEATHER,), 304),
|
||||
(_XAI_GROK_4_3, None, (_GET_WEATHER,), 283),
|
||||
],
|
||||
)
|
||||
async def test_responses_estimate_matches_the_billed_responses_prompt_with_the_shipped_framing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
system: str | None,
|
||||
tools: tuple[dict[str, object], ...],
|
||||
billed_prompt_tokens: int,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "billed",
|
||||
"litellm_params": {"model": model, "api_key": "unused", "api_base": _NATIVE_RESPONSES_BASE[model]},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
counted: Final = await count_request_tokens(
|
||||
request=TokenCountRequest(
|
||||
model="billed",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_responses_function(tool) for tool in tools] or None,
|
||||
system=system,
|
||||
),
|
||||
responses_request={"model": "billed", "input": "What is the weather in Paris?"},
|
||||
)
|
||||
|
||||
assert counted.count_source == "local_estimate"
|
||||
assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05
|
||||
|
||||
|
||||
def _single_deployment_router(litellm_params: dict[str, object], model_info: dict[str, object]) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "counted",
|
||||
"litellm_params": {"api_key": "unused", **litellm_params},
|
||||
"model_info": model_info,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def _count_with_router(
|
||||
monkeypatch: pytest.MonkeyPatch, router: Router, tools: tuple[dict[str, object], ...], responses: bool
|
||||
) -> int:
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
||||
counted: Final = await count_request_tokens(
|
||||
request=TokenCountRequest(
|
||||
model="counted",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=list(tools) or None,
|
||||
),
|
||||
responses_request={"model": "counted", "input": "What is the weather in Paris?"} if responses else None,
|
||||
)
|
||||
return counted.total_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tools", [(), (_GET_WEATHER,), _THREE_TOOLS, (_FILL_FORM,)])
|
||||
@pytest.mark.parametrize("responses", [False, True])
|
||||
async def test_a_custom_deployment_name_counts_like_the_base_model_it_names(
|
||||
monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...], responses: bool
|
||||
):
|
||||
api_base: Final = _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]
|
||||
custom_name: Final = _single_deployment_router(
|
||||
{"model": "azure_ai/prod-grok", "api_base": api_base}, {"base_model": _AZURE_GROK_4_1}
|
||||
)
|
||||
row_name: Final = _single_deployment_router({"model": _AZURE_GROK_4_1, "api_base": api_base}, {})
|
||||
|
||||
assert await _count_with_router(monkeypatch, custom_name, tools, responses) == await _count_with_router(
|
||||
monkeypatch, row_name, tools, responses
|
||||
)
|
||||
|
||||
|
||||
# usage.input_tokens that POST /v1/responses billed through Azure AI Foundry grok-4-1-fast-reasoning on 2026-10-29 for
|
||||
# the input "What is the weather in Paris?" with these function tools, strict left unset or sent as false or null,
|
||||
# the same on the .services.ai.azure.com and .openai.azure.com hosts
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "strict", "billed_prompt_tokens"),
|
||||
[
|
||||
((_GET_WEATHER,), {}, 159),
|
||||
((_GET_WEATHER,), {"strict": False}, 155),
|
||||
((_GET_WEATHER,), {"strict": None}, 155),
|
||||
(_THREE_TOOLS, {}, 257),
|
||||
(_get_weather_copies(5), {}, 345),
|
||||
(_get_weather_copies(10), {}, 575),
|
||||
(_get_weather_copies(10), {"strict": False}, 535),
|
||||
(_get_weather_copies(10), {"strict": None}, 535),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "model_info"),
|
||||
[
|
||||
({"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}),
|
||||
(
|
||||
{"model": "azure/grok-4-1-fast-reasoning", "api_base": "https://unused.openai.azure.com/openai/v1"},
|
||||
{"base_model": _AZURE_GROK_4_1},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_as_function_tools_are_added(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tools: tuple[dict[str, object], ...],
|
||||
strict: dict[str, bool | None],
|
||||
billed_prompt_tokens: int,
|
||||
litellm_params: dict[str, object],
|
||||
model_info: dict[str, object],
|
||||
):
|
||||
router: Final = _single_deployment_router(litellm_params, model_info)
|
||||
|
||||
counted: Final = await _count_with_router(
|
||||
monkeypatch, router, tuple(_responses_function(tool, **strict) for tool in tools), responses=True
|
||||
)
|
||||
|
||||
assert billed_prompt_tokens <= counted <= billed_prompt_tokens * 1.02
|
||||
|
||||
|
||||
# POST /v1/responses billed the same usage.input_tokens with strict unset and false on 2026-10-29, 283 for one tool
|
||||
# through xAI grok-4.3
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_responses_host_that_bills_an_unset_strict_like_false_counts_them_alike(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _single_deployment_router(
|
||||
{"model": _XAI_GROK_4_3, "api_base": _NATIVE_RESPONSES_BASE[_XAI_GROK_4_3]}, {}
|
||||
)
|
||||
|
||||
unset: Final = await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER),), responses=True)
|
||||
|
||||
assert unset == await _count_with_router(
|
||||
monkeypatch, router, (_responses_function(_GET_WEATHER, strict=False),), responses=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("built_in_tool", "responses"),
|
||||
[({"type": "web_search"}, True), ({"type": "web_search_20250305", "name": "web_search"}, False)],
|
||||
)
|
||||
async def test_a_request_whose_only_tools_are_built_in_counts_like_one_without_tools(
|
||||
monkeypatch: pytest.MonkeyPatch, built_in_tool: dict[str, object], responses: bool
|
||||
):
|
||||
router: Final = _single_deployment_router(
|
||||
{"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}
|
||||
)
|
||||
|
||||
assert await _count_with_router(monkeypatch, router, (built_in_tool,), responses) == await _count_with_router(
|
||||
monkeypatch, router, (), responses
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tools", [(), (_GET_WEATHER,), _THREE_TOOLS])
|
||||
async def test_a_responses_count_the_proxy_bridges_to_chat_completions_frames_like_the_chat_count(
|
||||
monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...]
|
||||
):
|
||||
bridged: Final = _single_deployment_router(
|
||||
{"model": _AZURE_GROK_4_1, "api_base": "https://unused.cognitiveservices.azure.com/models"}, {}
|
||||
)
|
||||
native: Final = _single_deployment_router(
|
||||
{"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}
|
||||
)
|
||||
chat_count: Final = await _count_with_router(monkeypatch, bridged, tools, responses=False)
|
||||
|
||||
assert await _count_with_router(monkeypatch, bridged, tools, responses=True) == chat_count
|
||||
assert await _count_with_router(monkeypatch, native, tools, responses=True) != chat_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("from_router_defaults", [False, True])
|
||||
async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frames_like_the_chat_count(
|
||||
monkeypatch: pytest.MonkeyPatch, from_router_defaults: bool
|
||||
):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "counted",
|
||||
"litellm_params": {
|
||||
"api_key": "unused",
|
||||
"model": _AZURE_GROK_4_1,
|
||||
"api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1],
|
||||
},
|
||||
"model_info": {},
|
||||
}
|
||||
],
|
||||
default_litellm_params={"use_chat_completions_api": True} if from_router_defaults else None,
|
||||
)
|
||||
chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False)
|
||||
|
||||
counted: Final = await count_request_tokens(
|
||||
request=TokenCountRequest(
|
||||
model="counted",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_GET_WEATHER],
|
||||
),
|
||||
responses_request={
|
||||
"model": "counted",
|
||||
"input": "What is the weather in Paris?",
|
||||
**({} if from_router_defaults else {"use_chat_completions_api": True}),
|
||||
},
|
||||
)
|
||||
|
||||
assert counted.total_tokens == chat_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_responses_count_frames_for_the_api_base_router_defaults_send_inference_to(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "counted",
|
||||
"litellm_params": {
|
||||
"api_key": "unused",
|
||||
"model": _AZURE_GROK_4_1,
|
||||
"api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1],
|
||||
},
|
||||
"model_info": {},
|
||||
}
|
||||
],
|
||||
default_litellm_params={"api_base": "https://unused.cognitiveservices.azure.com/models"},
|
||||
)
|
||||
chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False)
|
||||
|
||||
assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == chat_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_chat_completions_api", ["maybe", {"a": 1}])
|
||||
async def test_a_responses_count_with_a_malformed_bridge_flag_reads_it_the_way_the_responses_route_does(
|
||||
monkeypatch: pytest.MonkeyPatch, use_chat_completions_api: object
|
||||
):
|
||||
router: Final = _single_deployment_router(
|
||||
{"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}
|
||||
)
|
||||
chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False)
|
||||
|
||||
counted: Final = await count_request_tokens(
|
||||
request=TokenCountRequest(
|
||||
model="counted",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_GET_WEATHER],
|
||||
),
|
||||
responses_request={
|
||||
"model": "counted",
|
||||
"input": "What is the weather in Paris?",
|
||||
"use_chat_completions_api": use_chat_completions_api,
|
||||
},
|
||||
)
|
||||
|
||||
assert counted.total_tokens == chat_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_azure_responses_count_leaves_strict_unset_for_templates_not_measured_there(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _single_deployment_router(
|
||||
{"model": "azure_ai/deepseek-v4.1-flash", "api_base": "https://unused.services.ai.azure.com/models"}, {}
|
||||
)
|
||||
|
||||
async def counted(**strict: bool) -> int:
|
||||
return await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER, **strict),), True)
|
||||
|
||||
assert await counted() < await counted(strict=True)
|
||||
|
||||
|
||||
_KIMI_WEATHER: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
|
|
|
|||
|
|
@ -672,7 +672,7 @@ def test_pdf_opt_out_on_one_deployment_does_not_block_its_sibling(respx_mock: re
|
|||
|
||||
|
||||
def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None:
|
||||
"""A deployment's ``token_count_model`` and ``token_count_framing`` must not land on
|
||||
"""A deployment's ``token_count_model`` and framing settings must not land on
|
||||
the shared ``{provider}/{model}`` key, or a sibling deployment of that model would count with them."""
|
||||
from litellm.proxy.common_utils.token_count_settings import cost_map_entry
|
||||
|
||||
|
|
@ -681,6 +681,9 @@ def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None:
|
|||
configured: Final = {
|
||||
"token_count_model": {"model": "anthropic.claude-opus-5-5", "region": "us-west-2"},
|
||||
"token_count_framing": 7,
|
||||
"token_count_tool_framing": 3,
|
||||
"token_count_responses_framing": 5,
|
||||
"token_count_responses_tool_framing": 2,
|
||||
}
|
||||
builtin: Final = {field: cost_map_entry(backend_model).get(field) for field in configured}
|
||||
assert all(builtin[field] != value for field, value in configured.items()), builtin
|
||||
|
|
|
|||
|
|
@ -1178,6 +1178,9 @@ _COST_MAP_ROW_SCHEMA: Final = {
|
|||
"supports_vision": {"type": "boolean"},
|
||||
"supports_web_search": {"type": "boolean"},
|
||||
"token_count_framing": {"type": "integer"},
|
||||
"token_count_tool_framing": {"type": "integer"},
|
||||
"token_count_responses_framing": {"type": "integer"},
|
||||
"token_count_responses_tool_framing": {"type": "integer"},
|
||||
"token_count_model": {
|
||||
"type": "object",
|
||||
"properties": {"model": {"type": "string"}, "region": {"type": "string"}},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue