fix(count_tokens): verify azure provider without counter

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-10-21 00:11:38 +00:00
parent 4f182b97bf
commit 55bb1ae3f3
3 changed files with 35 additions and 5 deletions

View file

@ -123,7 +123,7 @@ def unreliable_local_estimate(
tools: Sequence[object] | None,
) -> str | None:
if _has_document_content_block(messages):
return f"{model} has no server-side count_tokens and LiteLLM cannot count document content locally"
return f"{model} returned no server-side token count and LiteLLM cannot count document content locally"
trigger: Final = "tools" if tools else "no_tools"
matched_rule: Final = next(
(

View file

@ -150,9 +150,7 @@ from litellm.proxy.common_utils.token_count_settings import (
TokenCountSettings,
cost_map_entry,
resolve_token_count_settings,
)
from litellm.proxy.common_utils.token_count_settings import (
unreliable_local_estimate as _unreliable_local_estimate,
unreliable_local_estimate,
)
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
from litellm.router_utils.access_windows import access_windows_config_error
@ -14114,7 +14112,7 @@ async def count_request_tokens(
return ladder
if call_endpoint is True:
unreliable_reason: Final = _unreliable_local_estimate(
unreliable_reason: Final = unreliable_local_estimate(
custom_llm_provider=custom_llm_provider,
model=model_to_use,
messages=messages,

View file

@ -2754,6 +2754,38 @@ async def test_count_request_tokens_rejects_only_known_unreliable_estimates(
assert response.count_source == "local_estimate"
@pytest.mark.asyncio
async def test_azure_ai_estimate_guard_runs_without_a_provider_counter(monkeypatch: pytest.MonkeyPatch) -> None:
router: Final = Router(
model_list=[
{
"model_name": "azure-ai-count-target",
"litellm_params": {
"model": "azure_ai/deepseek-v4-pro",
"api_key": "test-key",
"api_base": "https://count.test",
},
}
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
with pytest.raises(ProxyException) as raised:
await count_request_tokens(
request=TokenCountRequest(
model="azure-ai-count-target",
messages=[{"role": "user", "content": "Summarize."}],
tools=[{"name": "search"}],
),
call_endpoint=True,
)
assert raised.value.code == "400"
assert raised.value.type == "invalid_request_error"
assert raised.value.param == "model"
assert "deepseek-v4-pro" in raised.value.message
@pytest.mark.asyncio
async def test_document_block_keeps_successful_provider_count(monkeypatch: pytest.MonkeyPatch) -> None:
router: Final = Router(