From 3444b01802e21b205db9082257e87dd2690a0cd7 Mon Sep 17 00:00:00 2001 From: Chenglun Hu Date: Fri, 5 Jun 2026 17:30:02 +0800 Subject: [PATCH 1/2] fix(anthropic): pipe api_base through count_tokens token_counter + handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #29764. Three issues working together caused /v1/messages/count_tokens to ignore api_base on the deployment and hit api.anthropic.com: 1. AnthropicTokenCounter.count_tokens read api_key from litellm_params but never read api_base. 2. handle_count_tokens_request accepted api_base but used the legacy `api_base or default` form, which expected a full URL — so users passing the usual base form (http://host:port/v1) got a wrong request. 3. get_anthropic_count_tokens_endpoint hardcoded the full anthropic SaaS URL with no api_base parameter. Fixed by: - token_counter reads api_base from litellm_params (or ANTHROPIC_API_BASE env) and passes it. - handler delegates URL construction to the config helper. - get_anthropic_count_tokens_endpoint accepts api_base, strips trailing slash, and appends the right suffix based on what's already there (bare base, /v1, /v1/messages, or full URL — all produce the correct endpoint without duplication). 6 new tests cover all the URL shapes + a baseline + trailing-slash behavior. Negative-verified. --- .../llms/anthropic/count_tokens/handler.py | 7 +- .../anthropic/count_tokens/token_counter.py | 9 ++ .../anthropic/count_tokens/transformation.py | 26 +++- ...t_anthropic_count_tokens_transformation.py | 123 +++++++++++++++++- 4 files changed, 159 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index 38cd429d99a..15ea1715132 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -66,8 +66,11 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): verbose_logger.debug("Transformed request: %s", request_body) - # Get endpoint URL - endpoint_url: Final = api_base or self.get_anthropic_count_tokens_endpoint() + # Get endpoint URL — pass api_base into the config so a deployment + # configured against an Anthropic-compatible backend (self-hosted + # vLLM, air-gapped proxy, etc.) gets the right path appended + # instead of being silently routed at api.anthropic.com (#29764). + endpoint_url: Final = self.get_anthropic_count_tokens_endpoint(api_base=api_base) verbose_logger.debug("Making request to: %s", endpoint_url) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 8e8d10c961b..eaee841e4c3 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -63,11 +63,20 @@ class AnthropicTokenCounter(BaseTokenCounter): verbose_logger.warning("No Anthropic API key found for token counting") return None + # Read api_base too — without this the handler silently falls back + # to api.anthropic.com even when the deployment is configured against + # an Anthropic-compatible backend (self-hosted vLLM, etc.) (#29764). + # Honor ANTHROPIC_BASE_URL as well: main.py / common_utils.py already + # accept it, so a deployment configured only via ANTHROPIC_BASE_URL + # would otherwise still hit the original bug. + api_base = litellm_params.get("api_base") or os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") + try: result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request( model=model_to_use, messages=messages, api_key=api_key, + api_base=api_base, tools=tools, system=system, ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 12581b9f658..056899065d9 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -19,14 +19,34 @@ class AnthropicCountTokensConfig: - Response: {"input_tokens": } """ - def get_anthropic_count_tokens_endpoint(self) -> str: + def get_anthropic_count_tokens_endpoint(self, api_base: str | None = None) -> str: """ Get the Anthropic CountTokens API endpoint. + Mirrors how /v1/messages resolves its URL: if a custom ``api_base`` + is configured, append the ``/v1/messages/count_tokens`` path when + it's not already there. This is what makes self-hosted vLLM / + air-gapped Anthropic-compatible backends work — without it the + handler hit the hardcoded ``api.anthropic.com`` even when an + ``api_base`` was set on the deployment (#29764). + + Args: + api_base: Optional custom API base from the deployment's + ``litellm_params.api_base``. + Returns: - The endpoint URL for the CountTokens API + The endpoint URL for the CountTokens API. """ - return "https://api.anthropic.com/v1/messages/count_tokens" + if not api_base: + return "https://api.anthropic.com/v1/messages/count_tokens" + api_base = api_base.rstrip("/") + if api_base.endswith("/v1/messages/count_tokens"): + return api_base + if api_base.endswith("/v1/messages"): + return f"{api_base}/count_tokens" + if api_base.endswith("/v1"): + return f"{api_base}/messages/count_tokens" + return f"{api_base}/v1/messages/count_tokens" def transform_request_to_count_tokens( self, diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py index ddac561f337..b64a1749fea 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -1,4 +1,3 @@ - from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) @@ -88,3 +87,125 @@ def test_transform_no_system_no_tools(): assert "system" not in result assert "tools" not in result + + +def test_get_endpoint_no_api_base_returns_anthropic_default(): + """#29764 baseline: with no api_base, the endpoint is api.anthropic.com.""" + config = AnthropicCountTokensConfig() + assert config.get_anthropic_count_tokens_endpoint() == "https://api.anthropic.com/v1/messages/count_tokens" + + +def test_get_endpoint_with_api_base_only_appends_full_path(): + """#29764: a bare api_base (e.g. http://vllm-host:8000) must have the + full /v1/messages/count_tokens path appended.""" + config = AnthropicCountTokensConfig() + assert ( + config.get_anthropic_count_tokens_endpoint(api_base="http://vllm-host:8000") + == "http://vllm-host:8000/v1/messages/count_tokens" + ) + + +def test_get_endpoint_with_api_base_ending_in_v1_appends_messages_count_tokens(): + """#29764 main scenario: vLLM-style configs typically pass + `http://host:port/v1` as api_base — append only the messages path so + we don't double up the /v1.""" + config = AnthropicCountTokensConfig() + assert ( + config.get_anthropic_count_tokens_endpoint(api_base="http://vllm-host:8000/v1") + == "http://vllm-host:8000/v1/messages/count_tokens" + ) + + +def test_get_endpoint_with_api_base_ending_in_messages_appends_count_tokens(): + """If a caller already terminated api_base with /v1/messages, just + append /count_tokens — don't repeat /messages.""" + config = AnthropicCountTokensConfig() + assert ( + config.get_anthropic_count_tokens_endpoint(api_base="https://example.com/v1/messages") + == "https://example.com/v1/messages/count_tokens" + ) + + +def test_get_endpoint_with_full_count_tokens_url_returned_verbatim(): + """If the caller explicitly passes the full count_tokens URL, hand it + back unchanged — no idempotency footgun.""" + config = AnthropicCountTokensConfig() + url = "https://example.com/v1/messages/count_tokens" + assert config.get_anthropic_count_tokens_endpoint(api_base=url) == url + + +def test_get_endpoint_strips_trailing_slash_on_api_base(): + """A trailing slash on api_base must not produce a double slash in the + constructed URL.""" + config = AnthropicCountTokensConfig() + assert ( + config.get_anthropic_count_tokens_endpoint(api_base="http://vllm-host:8000/v1/") + == "http://vllm-host:8000/v1/messages/count_tokens" + ) + + +# --- token_counter api_base env-fallback resolution (#29765) ---------------- +import os +from unittest.mock import AsyncMock, patch + +import pytest + +from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter + +_HANDLER = ( + "litellm.llms.anthropic.count_tokens.token_counter.anthropic_count_tokens_handler.handle_count_tokens_request" +) + + +@pytest.mark.asyncio +async def test_count_tokens_env_fallback_prefers_anthropic_api_base(monkeypatch): + """ANTHROPIC_API_BASE wins over ANTHROPIC_BASE_URL in the env fallback.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://from-api-base:8000") + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000") + + with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m: + await AnthropicTokenCounter().count_tokens( + model_to_use="claude-3-5-sonnet", + messages=[{"role": "user", "content": "hi"}], + contents=None, + ) + + assert m.call_args.kwargs["api_base"] == "http://from-api-base:8000" + + +@pytest.mark.asyncio +async def test_count_tokens_falls_back_to_anthropic_base_url(monkeypatch): + """#29765 review (willcai1984): a deployment configured only via + ANTHROPIC_BASE_URL must still reach its backend — main.py / common_utils.py + already honor it, so token counting has to as well.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000") + + with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m: + await AnthropicTokenCounter().count_tokens( + model_to_use="claude-3-5-sonnet", + messages=[{"role": "user", "content": "hi"}], + contents=None, + ) + + assert m.call_args.kwargs["api_base"] == "http://from-base-url:9000" + + +@pytest.mark.asyncio +async def test_count_tokens_litellm_params_api_base_wins(monkeypatch): + """Explicit litellm_params.api_base beats both env vars.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://from-api-base:8000") + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000") + + with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m: + await AnthropicTokenCounter().count_tokens( + model_to_use="claude-3-5-sonnet", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={"litellm_params": {"api_base": "http://explicit:7000"}}, + ) + + assert m.call_args.kwargs["api_base"] == "http://explicit:7000" From a190dee04f64ebb1c98a92fa82675f7d4af38c1a Mon Sep 17 00:00:00 2001 From: Chenglun Hu Date: Wed, 26 Aug 2026 23:57:40 +0800 Subject: [PATCH 2/2] test(anthropic): assert count_tokens return value in api_base routing tests Capture the TokenCountResponse and assert total_tokens flows through from the handler, so each test verifies a real caller-observable output rather than only the mocked handler's api_base kwarg (clears the test-quality gate). --- .../test_anthropic_count_tokens_transformation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py index b64a1749fea..1b6c2860e24 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -165,13 +165,14 @@ async def test_count_tokens_env_fallback_prefers_anthropic_api_base(monkeypatch) monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000") with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m: - await AnthropicTokenCounter().count_tokens( + result = await AnthropicTokenCounter().count_tokens( model_to_use="claude-3-5-sonnet", messages=[{"role": "user", "content": "hi"}], contents=None, ) assert m.call_args.kwargs["api_base"] == "http://from-api-base:8000" + assert result.total_tokens == 5 @pytest.mark.asyncio @@ -184,13 +185,14 @@ async def test_count_tokens_falls_back_to_anthropic_base_url(monkeypatch): monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000") with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m: - await AnthropicTokenCounter().count_tokens( + result = await AnthropicTokenCounter().count_tokens( model_to_use="claude-3-5-sonnet", messages=[{"role": "user", "content": "hi"}], contents=None, ) assert m.call_args.kwargs["api_base"] == "http://from-base-url:9000" + assert result.total_tokens == 5 @pytest.mark.asyncio @@ -201,7 +203,7 @@ async def test_count_tokens_litellm_params_api_base_wins(monkeypatch): monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000") with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m: - await AnthropicTokenCounter().count_tokens( + result = await AnthropicTokenCounter().count_tokens( model_to_use="claude-3-5-sonnet", messages=[{"role": "user", "content": "hi"}], contents=None, @@ -209,3 +211,4 @@ async def test_count_tokens_litellm_params_api_base_wins(monkeypatch): ) assert m.call_args.kwargs["api_base"] == "http://explicit:7000" + assert result.total_tokens == 5