Merge a190dee04f into 1af7a403c6

litellm/llms/anthropic/count_tokens/handler.py

@@ -66,8 +66,11 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
 
             verbose_logger.debug("Transformed request: %s", request_body)
 
-            # Get endpoint URL
-            endpoint_url: Final = api_base or self.get_anthropic_count_tokens_endpoint()
+            # Get endpoint URL — pass api_base into the config so a deployment
+            # configured against an Anthropic-compatible backend (self-hosted
+            # vLLM, air-gapped proxy, etc.) gets the right path appended
+            # instead of being silently routed at api.anthropic.com (#29764).
+            endpoint_url: Final = self.get_anthropic_count_tokens_endpoint(api_base=api_base)
 
             verbose_logger.debug("Making request to: %s", endpoint_url)
 

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -63,11 +63,20 @@ class AnthropicTokenCounter(BaseTokenCounter):
             verbose_logger.warning("No Anthropic API key found for token counting")
             return None
 
+        # Read api_base too — without this the handler silently falls back
+        # to api.anthropic.com even when the deployment is configured against
+        # an Anthropic-compatible backend (self-hosted vLLM, etc.) (#29764).
+        # Honor ANTHROPIC_BASE_URL as well: main.py / common_utils.py already
+        # accept it, so a deployment configured only via ANTHROPIC_BASE_URL
+        # would otherwise still hit the original bug.
+        api_base = litellm_params.get("api_base") or os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL")
+
         try:
             result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
                 model=model_to_use,
                 messages=messages,
                 api_key=api_key,
+                api_base=api_base,
                 tools=tools,
                 system=system,
             )

litellm/llms/anthropic/count_tokens/transformation.py

@@ -19,14 +19,34 @@ class AnthropicCountTokensConfig:
     - Response: {"input_tokens": <number>}
     """
 
-    def get_anthropic_count_tokens_endpoint(self) -> str:
+    def get_anthropic_count_tokens_endpoint(self, api_base: str | None = None) -> str:
         """
         Get the Anthropic CountTokens API endpoint.
 
+        Mirrors how /v1/messages resolves its URL: if a custom ``api_base``
+        is configured, append the ``/v1/messages/count_tokens`` path when
+        it's not already there. This is what makes self-hosted vLLM /
+        air-gapped Anthropic-compatible backends work — without it the
+        handler hit the hardcoded ``api.anthropic.com`` even when an
+        ``api_base`` was set on the deployment (#29764).
+
+        Args:
+            api_base: Optional custom API base from the deployment's
+                ``litellm_params.api_base``.
+
         Returns:
-            The endpoint URL for the CountTokens API
+            The endpoint URL for the CountTokens API.
         """
-        return "https://api.anthropic.com/v1/messages/count_tokens"
+        if not api_base:
+            return "https://api.anthropic.com/v1/messages/count_tokens"
+        api_base = api_base.rstrip("/")
+        if api_base.endswith("/v1/messages/count_tokens"):
+            return api_base
+        if api_base.endswith("/v1/messages"):
+            return f"{api_base}/count_tokens"
+        if api_base.endswith("/v1"):
+            return f"{api_base}/messages/count_tokens"
+        return f"{api_base}/v1/messages/count_tokens"
 
     def transform_request_to_count_tokens(
         self,

tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py

@@ -1,4 +1,3 @@
-
 from litellm.llms.anthropic.count_tokens.transformation import (
     AnthropicCountTokensConfig,
 )
@@ -88,3 +87,128 @@ def test_transform_no_system_no_tools():
 
     assert "system" not in result
     assert "tools" not in result
+
+
+def test_get_endpoint_no_api_base_returns_anthropic_default():
+    """#29764 baseline: with no api_base, the endpoint is api.anthropic.com."""
+    config = AnthropicCountTokensConfig()
+    assert config.get_anthropic_count_tokens_endpoint() == "https://api.anthropic.com/v1/messages/count_tokens"
+
+
+def test_get_endpoint_with_api_base_only_appends_full_path():
+    """#29764: a bare api_base (e.g. http://vllm-host:8000) must have the
+    full /v1/messages/count_tokens path appended."""
+    config = AnthropicCountTokensConfig()
+    assert (
+        config.get_anthropic_count_tokens_endpoint(api_base="http://vllm-host:8000")
+        == "http://vllm-host:8000/v1/messages/count_tokens"
+    )
+
+
+def test_get_endpoint_with_api_base_ending_in_v1_appends_messages_count_tokens():
+    """#29764 main scenario: vLLM-style configs typically pass
+    `http://host:port/v1` as api_base — append only the messages path so
+    we don't double up the /v1."""
+    config = AnthropicCountTokensConfig()
+    assert (
+        config.get_anthropic_count_tokens_endpoint(api_base="http://vllm-host:8000/v1")
+        == "http://vllm-host:8000/v1/messages/count_tokens"
+    )
+
+
+def test_get_endpoint_with_api_base_ending_in_messages_appends_count_tokens():
+    """If a caller already terminated api_base with /v1/messages, just
+    append /count_tokens — don't repeat /messages."""
+    config = AnthropicCountTokensConfig()
+    assert (
+        config.get_anthropic_count_tokens_endpoint(api_base="https://example.com/v1/messages")
+        == "https://example.com/v1/messages/count_tokens"
+    )
+
+
+def test_get_endpoint_with_full_count_tokens_url_returned_verbatim():
+    """If the caller explicitly passes the full count_tokens URL, hand it
+    back unchanged — no idempotency footgun."""
+    config = AnthropicCountTokensConfig()
+    url = "https://example.com/v1/messages/count_tokens"
+    assert config.get_anthropic_count_tokens_endpoint(api_base=url) == url
+
+
+def test_get_endpoint_strips_trailing_slash_on_api_base():
+    """A trailing slash on api_base must not produce a double slash in the
+    constructed URL."""
+    config = AnthropicCountTokensConfig()
+    assert (
+        config.get_anthropic_count_tokens_endpoint(api_base="http://vllm-host:8000/v1/")
+        == "http://vllm-host:8000/v1/messages/count_tokens"
+    )
+
+
+# --- token_counter api_base env-fallback resolution (#29765) ----------------
+import os
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
+
+_HANDLER = (
+    "litellm.llms.anthropic.count_tokens.token_counter.anthropic_count_tokens_handler.handle_count_tokens_request"
+)
+
+
+@pytest.mark.asyncio
+async def test_count_tokens_env_fallback_prefers_anthropic_api_base(monkeypatch):
+    """ANTHROPIC_API_BASE wins over ANTHROPIC_BASE_URL in the env fallback."""
+    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
+    monkeypatch.setenv("ANTHROPIC_API_BASE", "http://from-api-base:8000")
+    monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000")
+
+    with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m:
+        result = await AnthropicTokenCounter().count_tokens(
+            model_to_use="claude-3-5-sonnet",
+            messages=[{"role": "user", "content": "hi"}],
+            contents=None,
+        )
+
+    assert m.call_args.kwargs["api_base"] == "http://from-api-base:8000"
+    assert result.total_tokens == 5
+
+
+@pytest.mark.asyncio
+async def test_count_tokens_falls_back_to_anthropic_base_url(monkeypatch):
+    """#29765 review (willcai1984): a deployment configured only via
+    ANTHROPIC_BASE_URL must still reach its backend — main.py / common_utils.py
+    already honor it, so token counting has to as well."""
+    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
+    monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
+    monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000")
+
+    with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m:
+        result = await AnthropicTokenCounter().count_tokens(
+            model_to_use="claude-3-5-sonnet",
+            messages=[{"role": "user", "content": "hi"}],
+            contents=None,
+        )
+
+    assert m.call_args.kwargs["api_base"] == "http://from-base-url:9000"
+    assert result.total_tokens == 5
+
+
+@pytest.mark.asyncio
+async def test_count_tokens_litellm_params_api_base_wins(monkeypatch):
+    """Explicit litellm_params.api_base beats both env vars."""
+    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
+    monkeypatch.setenv("ANTHROPIC_API_BASE", "http://from-api-base:8000")
+    monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://from-base-url:9000")
+
+    with patch(_HANDLER, new=AsyncMock(return_value={"input_tokens": 5})) as m:
+        result = await AnthropicTokenCounter().count_tokens(
+            model_to_use="claude-3-5-sonnet",
+            messages=[{"role": "user", "content": "hi"}],
+            contents=None,
+            deployment={"litellm_params": {"api_base": "http://explicit:7000"}},
+        )
+
+    assert m.call_args.kwargs["api_base"] == "http://explicit:7000"
+    assert result.total_tokens == 5