mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(count_tokens): verify azure provider without counter
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
4f182b97bf
commit
55bb1ae3f3
3 changed files with 35 additions and 5 deletions
|
|
@ -123,7 +123,7 @@ def unreliable_local_estimate(
|
|||
tools: Sequence[object] | None,
|
||||
) -> str | None:
|
||||
if _has_document_content_block(messages):
|
||||
return f"{model} has no server-side count_tokens and LiteLLM cannot count document content locally"
|
||||
return f"{model} returned no server-side token count and LiteLLM cannot count document content locally"
|
||||
trigger: Final = "tools" if tools else "no_tools"
|
||||
matched_rule: Final = next(
|
||||
(
|
||||
|
|
|
|||
|
|
@ -150,9 +150,7 @@ from litellm.proxy.common_utils.token_count_settings import (
|
|||
TokenCountSettings,
|
||||
cost_map_entry,
|
||||
resolve_token_count_settings,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_settings import (
|
||||
unreliable_local_estimate as _unreliable_local_estimate,
|
||||
unreliable_local_estimate,
|
||||
)
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
|
|
@ -14114,7 +14112,7 @@ async def count_request_tokens(
|
|||
return ladder
|
||||
|
||||
if call_endpoint is True:
|
||||
unreliable_reason: Final = _unreliable_local_estimate(
|
||||
unreliable_reason: Final = unreliable_local_estimate(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=model_to_use,
|
||||
messages=messages,
|
||||
|
|
|
|||
|
|
@ -2754,6 +2754,38 @@ async def test_count_request_tokens_rejects_only_known_unreliable_estimates(
|
|||
assert response.count_source == "local_estimate"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_ai_estimate_guard_runs_without_a_provider_counter(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "azure-ai-count-target",
|
||||
"litellm_params": {
|
||||
"model": "azure_ai/deepseek-v4-pro",
|
||||
"api_key": "test-key",
|
||||
"api_base": "https://count.test",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
|
||||
with pytest.raises(ProxyException) as raised:
|
||||
await count_request_tokens(
|
||||
request=TokenCountRequest(
|
||||
model="azure-ai-count-target",
|
||||
messages=[{"role": "user", "content": "Summarize."}],
|
||||
tools=[{"name": "search"}],
|
||||
),
|
||||
call_endpoint=True,
|
||||
)
|
||||
|
||||
assert raised.value.code == "400"
|
||||
assert raised.value.type == "invalid_request_error"
|
||||
assert raised.value.param == "model"
|
||||
assert "deepseek-v4-pro" in raised.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_document_block_keeps_successful_provider_count(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
router: Final = Router(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue