mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(count_tokens): count custom-named deployments with their base_model's tokenizer and tool template
This commit is contained in:
parent
5eb1042bbf
commit
32d4d83111
2 changed files with 103 additions and 5 deletions
|
|
@ -15185,12 +15185,22 @@ def _deployment_aws_region(deployment: _TokenCountDeployment) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _deployment_base_model(deployment: _TokenCountDeployment) -> str | None:
|
||||
return _non_empty_str(deployment.get("model_info", {}).get("base_model")) or _non_empty_str(
|
||||
deployment.get("litellm_params", {}).get("base_model")
|
||||
)
|
||||
|
||||
|
||||
def _unprefixed_base_model(deployment: _TokenCountDeployment) -> str | None:
|
||||
base_model: Final = _deployment_base_model(deployment)
|
||||
return None if base_model is None else base_model.split("/", 1)[-1]
|
||||
|
||||
|
||||
def _token_count_settings_model(deployment: _TokenCountDeployment, request_model: str) -> tuple[str, str | None]:
|
||||
model_info: Final = deployment.get("model_info", {})
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
base_model: Final = _non_empty_str(model_info.get("base_model")) or _non_empty_str(params.get("base_model"))
|
||||
base_model: Final = _deployment_base_model(deployment)
|
||||
if base_model is not None:
|
||||
return base_model, None
|
||||
params: Final = deployment.get("litellm_params", {})
|
||||
model: Final = _non_empty_str(params.get("model"))
|
||||
if model is None:
|
||||
return request_model, None
|
||||
|
|
@ -15546,7 +15556,7 @@ async def _count_request_tokens(
|
|||
count_source="local_estimate",
|
||||
)
|
||||
estimate: Final = await _local_token_estimate(
|
||||
model=deployment_openai_model(deployment_view) or model_to_use,
|
||||
model=deployment_openai_model(deployment_view) or _unprefixed_base_model(deployment_view) or model_to_use,
|
||||
prompt=prompt if counted_messages is None else None,
|
||||
messages=counted_messages,
|
||||
tools=counted_tools,
|
||||
|
|
|
|||
|
|
@ -3989,6 +3989,9 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r
|
|||
spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form"))
|
||||
|
||||
|
||||
_MULTILINGUAL_TEXT: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base_models_tokenizer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -4007,7 +4010,7 @@ async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base
|
|||
]
|
||||
),
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
messages: Final = [{"role": "user", "content": _MULTILINGUAL_TEXT}]
|
||||
|
||||
named: Final = await token_counter(request=TokenCountRequest(model="named", messages=messages))
|
||||
served: Final = await token_counter(request=TokenCountRequest(model="served", messages=messages))
|
||||
|
|
@ -4015,6 +4018,91 @@ async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base
|
|||
assert named.total_tokens == served.total_tokens
|
||||
|
||||
|
||||
_LOOKUP_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup_order",
|
||||
"description": "Look up an order by its id",
|
||||
"parameters": {"type": "object", "properties": {"order_id": {"type": "string"}}, "required": ["order_id"]},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_FLASH_BASE_MODEL: Final = "azure_ai/deepseek-v4-flash"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("served_model", "named_params", "named_model_info", "tools"),
|
||||
[
|
||||
pytest.param(
|
||||
"azure_ai/DeepSeek-V4-Flash-0731",
|
||||
{"model": "azure_ai/my-flash"},
|
||||
{"base_model": _FLASH_BASE_MODEL},
|
||||
[dict(_LOOKUP_TOOL)],
|
||||
id="model_info base_model, tool template",
|
||||
),
|
||||
pytest.param(
|
||||
"azure_ai/DeepSeek-V4-Flash-0731",
|
||||
{"model": "azure_ai/my-flash"},
|
||||
{"base_model": _FLASH_BASE_MODEL},
|
||||
None,
|
||||
id="model_info base_model, tokenizer",
|
||||
),
|
||||
pytest.param(
|
||||
"azure_ai/DeepSeek-V4-Flash-0731",
|
||||
{"model": "azure_ai/my-flash", "base_model": _FLASH_BASE_MODEL},
|
||||
{},
|
||||
[dict(_LOOKUP_TOOL)],
|
||||
id="litellm_params base_model",
|
||||
),
|
||||
pytest.param(
|
||||
"azure_ai/Qwen3.5-397B-A17B",
|
||||
{"model": "azure_ai/my-qwen"},
|
||||
{"base_model": "azure_ai/qwen3.5-397b-a17b"},
|
||||
[dict(_LOOKUP_TOOL)],
|
||||
id="qwen3.5 base_model, tool template",
|
||||
),
|
||||
pytest.param(
|
||||
"anthropic/claude-opus-5-5",
|
||||
{"model": "anthropic/my-opus"},
|
||||
{"base_model": "anthropic/claude-opus-5-5"},
|
||||
None,
|
||||
id="provider-prefixed claude base_model, tokenizer",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_token_counter_counts_a_custom_named_deployment_like_the_base_model_it_declares(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
served_model: str,
|
||||
named_params: Mapping[str, str],
|
||||
named_model_info: Mapping[str, str],
|
||||
tools: list[dict[str, JsonValue]] | None,
|
||||
) -> None:
|
||||
credentials: Final = {"api_key": "sk-test", "api_base": "https://example.services.ai.azure.com"}
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "named",
|
||||
"litellm_params": {**named_params, **credentials},
|
||||
"model_info": dict(named_model_info),
|
||||
},
|
||||
{"model_name": "served", "litellm_params": {"model": served_model, **credentials}},
|
||||
]
|
||||
),
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": _MULTILINGUAL_TEXT}]
|
||||
|
||||
named: Final = await token_counter(request=TokenCountRequest(model="named", messages=messages, tools=tools))
|
||||
served: Final = await token_counter(request=TokenCountRequest(model="served", messages=messages, tools=tools))
|
||||
|
||||
assert named.total_tokens == served.total_tokens
|
||||
|
||||
|
||||
_COUNTED_HELLO: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{"model": "count-log-gpt", "messages": [{"role": "user", "content": "Hello, how many tokens is this?"}]}
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue