From d7bde6c4bb7057b443a5a558afe9e61837f3eded Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 15:31:22 -0700 Subject: [PATCH] feat(proxy): add token_count_local_fallback to count locally when a provider count fails Since #46121 the proxy answers the provider's error when a count-tokens call fails, so a Bedrock Nova or Llama deployment answers 400 on /v1/messages/count_tokens where 1.92.0 answered a local estimate. litellm_settings.token_count_local_fallback: true brings that fallback back as an opt-in. The provider is still asked first, any failure falls back to the local estimate when local counting is allowed, the response says litellm_count_source: local_estimate, and one warning names the provider status. disable_token_counter still wins. Default off, so fail-closed stays the default Refs LIT-9763 --- litellm/__init__.py | 1 + litellm/proxy/proxy_server.py | 13 +- .../test_count_tokens_error_shapes_wire.py | 52 +++++++- tests/unit/proxy/test_proxy_token_counter.py | 113 +++++++++++++++++- 4 files changed, 172 insertions(+), 7 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index f6604a3b4e5..6eaf5187ff3 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -318,6 +318,7 @@ otel_tenant_destination_mode: str | None = None ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance disable_streaming_logging: bool = False disable_token_counter: bool = False +token_count_local_fallback: bool = False disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False disable_anthropic_gemini_context_caching_transform: bool = False diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 967cae27990..149c91fd672 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14779,8 +14779,9 @@ async def _provider_token_count( ) -> Union["TokenCountResponse", ProviderCountFailure, None]: """Count with the provider, using the token_count_model in its region when one is set. - Returns None when the count should be estimated locally, or the provider's failure when the framed local - estimate stands in for it and that failure is the answer if the estimate cannot read the request.""" + Returns None when the count should be estimated locally, or the provider's failure when the local estimate + stands in for it (token_count_local_fallback, or a framed model the provider cannot count) and that failure + is the answer if the estimate cannot read the request.""" count_model: Final = settings.count_model counting_model: Final = ( model_to_use @@ -14804,7 +14805,13 @@ async def _provider_token_count( "If that region is not enabled for the AWS account, enable it or set token_count_model in model_info" ) ) - if count_model is None and failure.cannot_count_model and settings.framing is not None and local_estimate_allowed: + if local_estimate_allowed and litellm.token_count_local_fallback is True: + verbose_proxy_logger.warning( + "Provider token count failed with status %s, counting locally because token_count_local_fallback is on", + failure.status_code, + ) + return failure + if local_estimate_allowed and count_model is None and failure.cannot_count_model and settings.framing is not None: return failure raise failure.to_proxy_exception() diff --git a/tests/integration/providers/test_count_tokens_error_shapes_wire.py b/tests/integration/providers/test_count_tokens_error_shapes_wire.py index 200f5e81673..6301e77d2ff 100644 --- a/tests/integration/providers/test_count_tokens_error_shapes_wire.py +++ b/tests/integration/providers/test_count_tokens_error_shapes_wire.py @@ -1,10 +1,12 @@ import json from collections.abc import Iterator +from pathlib import Path from typing import Final import anthropic import openai import pytest +import yaml from integration._support.client import Gateway, gateway_from_environment from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.wire import Reply, Request, wire_server @@ -13,7 +15,9 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_BASE_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" _MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Count this synthetic request."}] +_UPSTREAM_REJECT: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted reject"}} _PROVIDER_KEY_VARIABLES: Final = ( "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", @@ -58,6 +62,32 @@ def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Ite yield owned +def _local_fallback_config(path: Path) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(_BASE_CONFIG.read_text())) + litellm_settings: Final = _JSON_OBJECT.validate_python(base.get("litellm_settings") or {}) + path.write_text( + yaml.safe_dump({**base, "litellm_settings": {**litellm_settings, "token_count_local_fallback": True}}) + ) + return path + + +@pytest.fixture(scope="module") +def local_fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: + directory: Final = tmp_path_factory.mktemp("count-tokens-local-fallback") + with ( + gateway_from_environment() as gateway, + owned_proxy_process( + gateway, + directory, + {}, + config=_local_fallback_config(directory / "local-fallback.yaml"), + remove_environment=_PROVIDER_KEY_VARIABLES, + workers=1, + ) as owned, + ): + yield owned + + def _anthropic(gateway: Gateway) -> anthropic.Anthropic: return anthropic.Anthropic( api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client @@ -135,9 +165,8 @@ def test_azure_ai_anthropic_count_authenticates_with_an_entra_token(keyless_prox def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy: OwnedProxy) -> None: gateway: Final = keyless_proxy.gateway - upstream_error: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted reject"}} with ( - wire_server(lambda _request: Reply(status=400, body=json.dumps(upstream_error).encode())) as wire, + wire_server(lambda _request: Reply(status=400, body=json.dumps(_UPSTREAM_REJECT).encode())) as wire, gateway.scenario() as scenario, ): model: Final = scenario.model( @@ -156,7 +185,24 @@ def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy assert rejected.status_code == 400, rejected.text body: Final = _JSON_OBJECT.validate_json(rejected.content) assert set(body) <= {"type", "error", "request_id"}, body - assert body["error"] == upstream_error["error"], body + assert body["error"] == _UPSTREAM_REJECT["error"], body + assert len(wire.drain()) == 1 + + +def test_local_fallback_counts_a_rejected_anthropic_count_locally(local_fallback_proxy: OwnedProxy) -> None: + gateway: Final = local_fallback_proxy.gateway + with ( + wire_server(lambda _request: Reply(status=400, body=json.dumps(_UPSTREAM_REJECT).encode())) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-key", num_retries=0 + ) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES}) + assert counted.status_code == 200, counted.text + body: Final = _JSON_OBJECT.validate_json(counted.content) + assert body == {"input_tokens": body["input_tokens"], "litellm_count_source": "local_estimate"}, body + assert isinstance(body["input_tokens"], int) and body["input_tokens"] > 0, body assert len(wire.drain()) == 1 diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..1205d93713e 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -10,7 +10,8 @@ import json import logging import threading import traceback -from collections.abc import Callable, Coroutine, Mapping +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping +from contextlib import contextmanager from functools import partial from pathlib import Path from types import MappingProxyType @@ -3892,3 +3893,113 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r ] assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3 spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form")) + +_NOVA_COUNT_URL: Final = r"^https://bedrock-count-tokens\.test/model/.+/count-tokens$" +_NOVA_CANNOT_COUNT: Final = "The provided model doesn't support counting tokens." + + +@contextmanager +def _nova_count_api( + monkeypatch: pytest.MonkeyPatch, + status_code: int, + local_fallback: bool, + model_info: dict[str, JsonValue] | None = None, +) -> Iterator[respx.Route]: + monkeypatch.setattr(litellm, "token_count_local_fallback", local_fallback) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "nova", + "litellm_params": { + "model": "bedrock/us.amazon.nova-pro-v1:0", + "api_base": "https://bedrock-count-tokens.test", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-east-1", + }, + "model_info": model_info or {}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + try: + with respx.mock(assert_all_called=False) as upstream: + yield upstream.post(url__regex=_NOVA_COUNT_URL).respond(status_code, json={"message": _NOVA_CANNOT_COUNT}) + finally: + litellm.in_memory_llm_clients_cache.flush_cache() + + +def _count_nova(messages_format: TokenCountMessagesFormat = "openai") -> Awaitable[TokenCountResponse]: + return count_request_tokens( + TokenCountRequest(model="nova", messages=[{"role": "user", "content": "hello there"}]), + call_endpoint=True, + messages_format=messages_format, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("messages_format", ("anthropic", "openai")) +async def test_local_fallback_counts_locally_when_the_provider_cannot_count_the_model( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, messages_format: TokenCountMessagesFormat +) -> None: + verbose_proxy_logger.addHandler(caplog.handler) + try: + with ( + _nova_count_api(monkeypatch, 400, local_fallback=True) as route, + caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name), + ): + counted: Final = await _count_nova(messages_format) + finally: + verbose_proxy_logger.removeHandler(caplog.handler) + + assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1) + warnings: Final = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("status 400" in message and "token_count_local_fallback" in message for message in warnings) + assert not any("hello there" in message for message in warnings) + + +@pytest.mark.asyncio +async def test_local_fallback_counts_locally_on_any_provider_status(monkeypatch: pytest.MonkeyPatch) -> None: + with _nova_count_api(monkeypatch, 401, local_fallback=True) as route: + counted: Final = await _count_nova() + + assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1) + + +@pytest.mark.asyncio +async def test_without_local_fallback_the_provider_count_error_is_the_answer(monkeypatch: pytest.MonkeyPatch) -> None: + with _nova_count_api(monkeypatch, 400, local_fallback=False) as route, pytest.raises(ProxyException) as raised: + await _count_nova() + + assert (raised.value.code, route.call_count) == ("400", 1) + assert _NOVA_CANNOT_COUNT in raised.value.message + + +@pytest.mark.asyncio +async def test_disabled_token_counter_wins_over_local_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_token_counter", True) + + with _nova_count_api(monkeypatch, 400, local_fallback=True), pytest.raises(ProxyException) as raised: + await _count_nova() + + assert raised.value.code == "400" + assert _NOVA_CANNOT_COUNT in raised.value.message + + +@pytest.mark.asyncio +async def test_local_fallback_also_covers_a_token_count_model_failing_in_its_own_region( + monkeypatch: pytest.MonkeyPatch, +) -> None: + count_model: Final[dict[str, JsonValue]] = { + "token_count_model": {"model": "us.amazon.nova-premier-v1:0", "region": "us-east-1"} + } + with _nova_count_api(monkeypatch, 400, local_fallback=True, model_info=count_model) as route: + counted: Final = await _count_nova() + + assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1) + assert "nova-premier" in route.calls.last.request.url.path