mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat(proxy): add token_count_local_fallback to count locally when a provider count fails
Since #46121 the proxy answers the provider's error when a count-tokens call fails, so a Bedrock Nova or Llama deployment answers 400 on /v1/messages/count_tokens where 1.92.0 answered a local estimate. litellm_settings.token_count_local_fallback: true brings that fallback back as an opt-in. The provider is still asked first, any failure falls back to the local estimate when local counting is allowed, the response says litellm_count_source: local_estimate, and one warning names the provider status. disable_token_counter still wins. Default off, so fail-closed stays the default Refs LIT-9763
This commit is contained in:
parent
730db4d228
commit
d7bde6c4bb
4 changed files with 172 additions and 7 deletions
|
|
@ -318,6 +318,7 @@ otel_tenant_destination_mode: str | None = None
|
|||
ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance
|
||||
disable_streaming_logging: bool = False
|
||||
disable_token_counter: bool = False
|
||||
token_count_local_fallback: bool = False
|
||||
disable_add_transform_inline_image_block: bool = False
|
||||
disable_add_user_agent_to_request_tags: bool = False
|
||||
disable_anthropic_gemini_context_caching_transform: bool = False
|
||||
|
|
|
|||
|
|
@ -14779,8 +14779,9 @@ async def _provider_token_count(
|
|||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
"""Count with the provider, using the token_count_model in its region when one is set.
|
||||
|
||||
Returns None when the count should be estimated locally, or the provider's failure when the framed local
|
||||
estimate stands in for it and that failure is the answer if the estimate cannot read the request."""
|
||||
Returns None when the count should be estimated locally, or the provider's failure when the local estimate
|
||||
stands in for it (token_count_local_fallback, or a framed model the provider cannot count) and that failure
|
||||
is the answer if the estimate cannot read the request."""
|
||||
count_model: Final = settings.count_model
|
||||
counting_model: Final = (
|
||||
model_to_use
|
||||
|
|
@ -14804,7 +14805,13 @@ async def _provider_token_count(
|
|||
"If that region is not enabled for the AWS account, enable it or set token_count_model in model_info"
|
||||
)
|
||||
)
|
||||
if count_model is None and failure.cannot_count_model and settings.framing is not None and local_estimate_allowed:
|
||||
if local_estimate_allowed and litellm.token_count_local_fallback is True:
|
||||
verbose_proxy_logger.warning(
|
||||
"Provider token count failed with status %s, counting locally because token_count_local_fallback is on",
|
||||
failure.status_code,
|
||||
)
|
||||
return failure
|
||||
if local_estimate_allowed and count_model is None and failure.cannot_count_model and settings.framing is not None:
|
||||
return failure
|
||||
raise failure.to_proxy_exception()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import json
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import openai
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, gateway_from_environment
|
||||
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
|
@ -13,7 +15,9 @@ from pydantic import JsonValue, TypeAdapter
|
|||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_BASE_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
|
||||
_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Count this synthetic request."}]
|
||||
_UPSTREAM_REJECT: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted reject"}}
|
||||
_PROVIDER_KEY_VARIABLES: Final = (
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_AUTH_TOKEN",
|
||||
|
|
@ -58,6 +62,32 @@ def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Ite
|
|||
yield owned
|
||||
|
||||
|
||||
def _local_fallback_config(path: Path) -> Path:
|
||||
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(_BASE_CONFIG.read_text()))
|
||||
litellm_settings: Final = _JSON_OBJECT.validate_python(base.get("litellm_settings") or {})
|
||||
path.write_text(
|
||||
yaml.safe_dump({**base, "litellm_settings": {**litellm_settings, "token_count_local_fallback": True}})
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def local_fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
|
||||
directory: Final = tmp_path_factory.mktemp("count-tokens-local-fallback")
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
owned_proxy_process(
|
||||
gateway,
|
||||
directory,
|
||||
{},
|
||||
config=_local_fallback_config(directory / "local-fallback.yaml"),
|
||||
remove_environment=_PROVIDER_KEY_VARIABLES,
|
||||
workers=1,
|
||||
) as owned,
|
||||
):
|
||||
yield owned
|
||||
|
||||
|
||||
def _anthropic(gateway: Gateway) -> anthropic.Anthropic:
|
||||
return anthropic.Anthropic(
|
||||
api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client
|
||||
|
|
@ -135,9 +165,8 @@ def test_azure_ai_anthropic_count_authenticates_with_an_entra_token(keyless_prox
|
|||
|
||||
def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy: OwnedProxy) -> None:
|
||||
gateway: Final = keyless_proxy.gateway
|
||||
upstream_error: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted reject"}}
|
||||
with (
|
||||
wire_server(lambda _request: Reply(status=400, body=json.dumps(upstream_error).encode())) as wire,
|
||||
wire_server(lambda _request: Reply(status=400, body=json.dumps(_UPSTREAM_REJECT).encode())) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
|
|
@ -156,7 +185,24 @@ def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy
|
|||
assert rejected.status_code == 400, rejected.text
|
||||
body: Final = _JSON_OBJECT.validate_json(rejected.content)
|
||||
assert set(body) <= {"type", "error", "request_id"}, body
|
||||
assert body["error"] == upstream_error["error"], body
|
||||
assert body["error"] == _UPSTREAM_REJECT["error"], body
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def test_local_fallback_counts_a_rejected_anthropic_count_locally(local_fallback_proxy: OwnedProxy) -> None:
|
||||
gateway: Final = local_fallback_proxy.gateway
|
||||
with (
|
||||
wire_server(lambda _request: Reply(status=400, body=json.dumps(_UPSTREAM_REJECT).encode())) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-key", num_retries=0
|
||||
)
|
||||
counted: Final = gateway.request("POST", "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES})
|
||||
assert counted.status_code == 200, counted.text
|
||||
body: Final = _JSON_OBJECT.validate_json(counted.content)
|
||||
assert body == {"input_tokens": body["input_tokens"], "litellm_count_source": "local_estimate"}, body
|
||||
assert isinstance(body["input_tokens"], int) and body["input_tokens"] > 0, body
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ import json
|
|||
import logging
|
||||
import threading
|
||||
import traceback
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
|
|
@ -3892,3 +3893,113 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r
|
|||
]
|
||||
assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3
|
||||
spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form"))
|
||||
|
||||
_NOVA_COUNT_URL: Final = r"^https://bedrock-count-tokens\.test/model/.+/count-tokens$"
|
||||
_NOVA_CANNOT_COUNT: Final = "The provided model doesn't support counting tokens."
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _nova_count_api(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
status_code: int,
|
||||
local_fallback: bool,
|
||||
model_info: dict[str, JsonValue] | None = None,
|
||||
) -> Iterator[respx.Route]:
|
||||
monkeypatch.setattr(litellm, "token_count_local_fallback", local_fallback)
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "nova",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/us.amazon.nova-pro-v1:0",
|
||||
"api_base": "https://bedrock-count-tokens.test",
|
||||
"aws_access_key_id": "test-access-key",
|
||||
"aws_secret_access_key": "test-secret-key",
|
||||
"aws_region_name": "us-east-1",
|
||||
},
|
||||
"model_info": model_info or {},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
try:
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
yield upstream.post(url__regex=_NOVA_COUNT_URL).respond(status_code, json={"message": _NOVA_CANNOT_COUNT})
|
||||
finally:
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
def _count_nova(messages_format: TokenCountMessagesFormat = "openai") -> Awaitable[TokenCountResponse]:
|
||||
return count_request_tokens(
|
||||
TokenCountRequest(model="nova", messages=[{"role": "user", "content": "hello there"}]),
|
||||
call_endpoint=True,
|
||||
messages_format=messages_format,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("messages_format", ("anthropic", "openai"))
|
||||
async def test_local_fallback_counts_locally_when_the_provider_cannot_count_the_model(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, messages_format: TokenCountMessagesFormat
|
||||
) -> None:
|
||||
verbose_proxy_logger.addHandler(caplog.handler)
|
||||
try:
|
||||
with (
|
||||
_nova_count_api(monkeypatch, 400, local_fallback=True) as route,
|
||||
caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name),
|
||||
):
|
||||
counted: Final = await _count_nova(messages_format)
|
||||
finally:
|
||||
verbose_proxy_logger.removeHandler(caplog.handler)
|
||||
|
||||
assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1)
|
||||
warnings: Final = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert any("status 400" in message and "token_count_local_fallback" in message for message in warnings)
|
||||
assert not any("hello there" in message for message in warnings)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_fallback_counts_locally_on_any_provider_status(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with _nova_count_api(monkeypatch, 401, local_fallback=True) as route:
|
||||
counted: Final = await _count_nova()
|
||||
|
||||
assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_without_local_fallback_the_provider_count_error_is_the_answer(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with _nova_count_api(monkeypatch, 400, local_fallback=False) as route, pytest.raises(ProxyException) as raised:
|
||||
await _count_nova()
|
||||
|
||||
assert (raised.value.code, route.call_count) == ("400", 1)
|
||||
assert _NOVA_CANNOT_COUNT in raised.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_token_counter_wins_over_local_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_token_counter", True)
|
||||
|
||||
with _nova_count_api(monkeypatch, 400, local_fallback=True), pytest.raises(ProxyException) as raised:
|
||||
await _count_nova()
|
||||
|
||||
assert raised.value.code == "400"
|
||||
assert _NOVA_CANNOT_COUNT in raised.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_fallback_also_covers_a_token_count_model_failing_in_its_own_region(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
count_model: Final[dict[str, JsonValue]] = {
|
||||
"token_count_model": {"model": "us.amazon.nova-premier-v1:0", "region": "us-east-1"}
|
||||
}
|
||||
with _nova_count_api(monkeypatch, 400, local_fallback=True, model_info=count_model) as route:
|
||||
counted: Final = await _count_nova()
|
||||
|
||||
assert (counted.count_source, counted.total_tokens > 0, route.call_count) == ("local_estimate", True, 1)
|
||||
assert "nova-premier" in route.calls.last.request.url.path
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue