feat(proxy): add token_count_local_fallback to count locally when a provider count fails

Since #46121 the proxy answers the provider's error when a count-tokens call fails,
so a Bedrock Nova or Llama deployment answers 400 on /v1/messages/count_tokens where
1.92.0 answered a local estimate. litellm_settings.token_count_local_fallback: true
brings that fallback back as an opt-in. The provider is still asked first, any failure
falls back to the local estimate when local counting is allowed, the response says
litellm_count_source: local_estimate, and one warning names the provider status.
disable_token_counter still wins. Default off, so fail-closed stays the default

Refs LIT-9763
This commit is contained in:
ryan-crabbe-berri 2026-10-20 15:31:22 -07:00
parent 730db4d228
commit d7bde6c4bb
4 changed files with 172 additions and 7 deletions

View file

@ -318,6 +318,7 @@ otel_tenant_destination_mode: str | None = None
ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance
disable_streaming_logging: bool = False
disable_token_counter: bool = False
token_count_local_fallback: bool = False
disable_add_transform_inline_image_block: bool = False
disable_add_user_agent_to_request_tags: bool = False
disable_anthropic_gemini_context_caching_transform: bool = False

View file

@ -14779,8 +14779,9 @@ async def _provider_token_count(
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
"""Count with the provider, using the token_count_model in its region when one is set.
Returns None when the count should be estimated locally, or the provider's failure when the framed local
estimate stands in for it and that failure is the answer if the estimate cannot read the request."""
Returns None when the count should be estimated locally, or the provider's failure when the local estimate
stands in for it (token_count_local_fallback, or a framed model the provider cannot count) and that failure
is the answer if the estimate cannot read the request."""
count_model: Final = settings.count_model
counting_model: Final = (
model_to_use
@ -14804,7 +14805,13 @@ async def _provider_token_count(
"If that region is not enabled for the AWS account, enable it or set token_count_model in model_info"
)
)
if count_model is None and failure.cannot_count_model and settings.framing is not None and local_estimate_allowed:
if local_estimate_allowed and litellm.token_count_local_fallback is True:
verbose_proxy_logger.warning(
"Provider token count failed with status %s, counting locally because token_count_local_fallback is on",
failure.status_code,
)
return failure
if local_estimate_allowed and count_model is None and failure.cannot_count_model and settings.framing is not None:
return failure
raise failure.to_proxy_exception()

View file

@ -1,10 +1,12 @@
import json
from collections.abc import Iterator
from pathlib import Path
from typing import Final
import anthropic
import openai
import pytest
import yaml
from integration._support.client import Gateway, gateway_from_environment
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, wire_server
@ -13,7 +15,9 @@ from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_BASE_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Count this synthetic request."}]
_UPSTREAM_REJECT: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted reject"}}
_PROVIDER_KEY_VARIABLES: Final = (
"ANTHROPIC_API_KEY",
"ANTHROPIC_AUTH_TOKEN",
@ -58,6 +62,32 @@ def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Ite
yield owned
def _local_fallback_config(path: Path) -> Path:
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(_BASE_CONFIG.read_text()))
litellm_settings: Final = _JSON_OBJECT.validate_python(base.get("litellm_settings") or {})
path.write_text(
yaml.safe_dump({**base, "litellm_settings": {**litellm_settings, "token_count_local_fallback": True}})
)
return path
@pytest.fixture(scope="module")
def local_fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
directory: Final = tmp_path_factory.mktemp("count-tokens-local-fallback")
with (
gateway_from_environment() as gateway,
owned_proxy_process(
gateway,
directory,
{},
config=_local_fallback_config(directory / "local-fallback.yaml"),
remove_environment=_PROVIDER_KEY_VARIABLES,
workers=1,
) as owned,
):
yield owned
def _anthropic(gateway: Gateway) -> anthropic.Anthropic:
return anthropic.Anthropic(
api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client
@ -135,9 +165,8 @@ def test_azure_ai_anthropic_count_authenticates_with_an_entra_token(keyless_prox
def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy: OwnedProxy) -> None:
gateway: Final = keyless_proxy.gateway
upstream_error: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted reject"}}
with (
wire_server(lambda _request: Reply(status=400, body=json.dumps(upstream_error).encode())) as wire,
wire_server(lambda _request: Reply(status=400, body=json.dumps(_UPSTREAM_REJECT).encode())) as wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(
@ -156,7 +185,24 @@ def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy
assert rejected.status_code == 400, rejected.text
body: Final = _JSON_OBJECT.validate_json(rejected.content)
assert set(body) <= {"type", "error", "request_id"}, body
assert body["error"] == upstream_error["error"], body
assert body["error"] == _UPSTREAM_REJECT["error"], body
assert len(wire.drain()) == 1
def test_local_fallback_counts_a_rejected_anthropic_count_locally(local_fallback_proxy: OwnedProxy) -> None:
gateway: Final = local_fallback_proxy.gateway
with (
wire_server(lambda _request: Reply(status=400, body=json.dumps(_UPSTREAM_REJECT).encode())) as wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(
model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-key", num_retries=0
)
counted: Final = gateway.request("POST", "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES})
assert counted.status_code == 200, counted.text
body: Final = _JSON_OBJECT.validate_json(counted.content)
assert body == {"input_tokens": body["input_tokens"], "litellm_count_source": "local_estimate"}, body
assert isinstance(body["input_tokens"], int) and body["input_tokens"] > 0, body
assert len(wire.drain()) == 1

View file

@ -10,7 +10,8 @@ import json
import logging
import threading
import traceback
from collections.abc import Callable, Coroutine, Mapping
from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from types import MappingProxyType
@ -3892,3 +3893,113 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r
]
assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3
spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form"))
_NOVA_COUNT_URL: Final = r"^https://bedrock-count-tokens\.test/model/.+/count-tokens$"
_NOVA_CANNOT_COUNT: Final = "The provided model doesn't support counting tokens."
@contextmanager
def _nova_count_api(
monkeypatch: pytest.MonkeyPatch,
status_code: int,
local_fallback: bool,
model_info: dict[str, JsonValue] | None = None,
) -> Iterator[respx.Route]:
monkeypatch.setattr(litellm, "token_count_local_fallback", local_fallback)
monkeypatch.setattr(
litellm.proxy.proxy_server,
"llm_router",
Router(
model_list=[
{
"model_name": "nova",
"litellm_params": {
"model": "bedrock/us.amazon.nova-pro-v1:0",
"api_base": "https://bedrock-count-tokens.test",
"aws_access_key_id": "test-access-key",
"aws_secret_access_key": "test-secret-key",
"aws_region_name": "us-east-1",
},
"model_info": model_info or {},
}
]
),
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
try:
with respx.mock(assert_all_called=False) as upstream:
yield upstream.post(url__regex=_NOVA_COUNT_URL).respond(status_code, json={"message": _NOVA_CANNOT_COUNT})
finally:
litellm.in_memory_llm_clients_cache.flush_cache()
def _count_nova(messages_format: TokenCountMessagesFormat = "openai") -> Awaitable[TokenCountResponse]:
return count_request_tokens(
TokenCountRequest(model="nova", messages=[{"role": "user", "content": "hello there"}]),
call_endpoint=True,
messages_format=messages_format,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("messages_format", ("anthropic", "openai"))
async def test_local_fallback_counts_locally_when_the_provider_cannot_count_the_model(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, messages_format: TokenCountMessagesFormat
) -> None:
verbose_proxy_logger.addHandler(caplog.handler)
try:
with (
_nova_count_api(monkeypatch, 400, local_fallback=True) as route,
caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name),
):
counted: Final = await _count_nova(messages_format)
finally:
verbose_proxy_logger.removeHandler(caplog.handler)
assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1)
warnings: Final = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
assert any("status 400" in message and "token_count_local_fallback" in message for message in warnings)
assert not any("hello there" in message for message in warnings)
@pytest.mark.asyncio
async def test_local_fallback_counts_locally_on_any_provider_status(monkeypatch: pytest.MonkeyPatch) -> None:
with _nova_count_api(monkeypatch, 401, local_fallback=True) as route:
counted: Final = await _count_nova()
assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1)
@pytest.mark.asyncio
async def test_without_local_fallback_the_provider_count_error_is_the_answer(monkeypatch: pytest.MonkeyPatch) -> None:
with _nova_count_api(monkeypatch, 400, local_fallback=False) as route, pytest.raises(ProxyException) as raised:
await _count_nova()
assert (raised.value.code, route.call_count) == ("400", 1)
assert _NOVA_CANNOT_COUNT in raised.value.message
@pytest.mark.asyncio
async def test_disabled_token_counter_wins_over_local_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_token_counter", True)
with _nova_count_api(monkeypatch, 400, local_fallback=True), pytest.raises(ProxyException) as raised:
await _count_nova()
assert raised.value.code == "400"
assert _NOVA_CANNOT_COUNT in raised.value.message
@pytest.mark.asyncio
async def test_local_fallback_also_covers_a_token_count_model_failing_in_its_own_region(
monkeypatch: pytest.MonkeyPatch,
) -> None:
count_model: Final[dict[str, JsonValue]] = {
"token_count_model": {"model": "us.amazon.nova-premier-v1:0", "region": "us-east-1"}
}
with _nova_count_api(monkeypatch, 400, local_fallback=True, model_info=count_model) as route:
counted: Final = await _count_nova()
assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1)
assert "nova-premier" in route.calls.last.request.url.path