This commit is contained in:
Mateo Wang 2026-10-27 02:23:02 +00:00 • committed by GitHub
commit d43708fe99
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 306 additions and 25 deletions

View file

@ -8,7 +8,7 @@ from typing import Final
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import JSONResponse
from pydantic import JsonValue, ValidationError
from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -48,7 +48,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
openai_error_type,
with_litellm_call_id,
)
from litellm.proxy.common_utils.token_count_settings import count_source_fields
from litellm.proxy.common_utils.token_count_settings import count_source_fields, counts_with_billed_request
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
from litellm.router import Router
@ -169,6 +169,20 @@ async def anthropic_response(
This was a BETA endpoint that calls 100+ LLMs in the anthropic format.
"""
return await _anthropic_messages_response(
ProxyBaseLLMRequestProcessing(data=await read_request_body(request=request)),
fastapi_response,
request,
user_api_key_dict,
)
async def _anthropic_messages_response(
base_llm_response_processor: ProxyBaseLLMRequestProcessing,
fastapi_response: Response,
request: Request,
user_api_key_dict: UserAPIKeyAuth,
) -> object:
from litellm.proxy.proxy_server import (
general_settings,
llm_router,
@ -182,8 +196,7 @@ async def anthropic_response(
version,
)
data: Final = await read_request_body(request=request)
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
data: Final = base_llm_response_processor.data
try:
result: Final = await base_llm_response_processor.base_process_llm_request(
request=request,
@ -244,7 +257,7 @@ async def anthropic_response(
usage=_usage,
)
if data.get("stream", None) is not None and data["stream"] is True:
if "stream" in data and data["stream"] is True:
# For streaming, use the standard SSE data generator
async def _passthrough_stream_generator():
yield _anthropic_response
@ -334,6 +347,43 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
class _BilledUsage(BaseModel):
model_config = ConfigDict(frozen=True)
input_tokens: int
cache_read_input_tokens: int | None = None
cache_creation_input_tokens: int | None = None
@property
def billed_input_tokens(self) -> int:
return self.input_tokens + (self.cache_read_input_tokens or 0) + (self.cache_creation_input_tokens or 0)
class _BilledResponse(BaseModel):
model_config = ConfigDict(frozen=True)
usage: _BilledUsage
def _counts_with_billed_request(llm_router: Router | None, model: str, team_id: str | None) -> bool:
if llm_router is None:
return False
return counts_with_billed_request(llm_router.get_model_list(model_name=model, team_id=team_id) or ())
async def _billed_input_tokens(
probe: ProxyBaseLLMRequestProcessing,
fastapi_response: Response,
request: Request,
user_api_key_dict: UserAPIKeyAuth,
) -> object:
response: Final = await _anthropic_messages_response(probe, fastapi_response, request, user_api_key_dict)
if isinstance(response, Response):
return response
billed: Final = _BilledResponse.model_validate(response, from_attributes=True)
return {"input_tokens": billed.usage.billed_input_tokens, "litellm_count_source": "billed_request"}
class _CountTokensBodyOptions(TypedDict):
output_format: ReadOnly[JsonValue]
context_management: ReadOnly[JsonValue]
@ -357,6 +407,7 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
dependencies=[Depends(user_api_key_auth)],
)
async def count_tokens(
fastapi_response: Response,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # Used for auth
):
@ -397,6 +448,18 @@ async def count_tokens(
if not messages:
raise HTTPException(status_code=400, detail="messages parameter is required")
if isinstance(model_name, str) and _counts_with_billed_request(
llm_router, model_name, user_api_key_dict.team_id
):
return await _billed_input_tokens(
ProxyBaseLLMRequestProcessing(
data={**request_data, "max_tokens": 1, "stream": False, "disable_fallbacks": True}
),
fastapi_response,
request,
user_api_key_dict,
)
auto_drop_params: Final = should_auto_drop_params_for_agentic_cli(
request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
)

View file

@ -1,4 +1,4 @@
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Final, Literal
@ -176,6 +176,13 @@ def resolve_token_count_settings(
)
def counts_with_billed_request(deployments: Sequence[Mapping[str, object]]) -> bool:
return bool(deployments) and all(
(as_str_mapping(deployment.get("model_info")) or {}).get("token_count_billed_request") is True
for deployment in deployments
)
def count_source_fields(response: TokenCountResponse) -> TokenCountSourceFields:
match response.count_source:
case None:

View file

@ -2,7 +2,7 @@ from typing import Literal, TypeAlias
from typing_extensions import NotRequired, ReadOnly, TypedDict
TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate"]
TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate", "billed_request"]
TokenCountTokenizer: TypeAlias = Literal["kimi_k3"]
TokenCountProvider: TypeAlias = Literal["bedrock", "bedrock_mantle"]

View file

@ -5,12 +5,18 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec
import json
import logging
import unittest
from collections.abc import Mapping
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from fastapi import Request, Response
from fastapi.testclient import TestClient
from starlette.types import Message
import litellm
from litellm import Router
from litellm._logging import verbose_proxy_logger
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
@ -462,7 +468,7 @@ class TestErrorLogCarriesCallId:
), # test-quality-ok: module global imported at call time; the test targets the endpoint's except block
caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"),
):
response = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
response = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
assert response.status_code == 500
record = self._error_record(caplog)
@ -514,7 +520,7 @@ async def test_count_tokens_answers_400_for_malformed_tools_without_counting(mod
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts it is never reached
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
assert response.status_code == 400
assert json.loads(response.body)["error"]["type"] == "invalid_request_error"
@ -537,7 +543,7 @@ async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
assert response == {"input_tokens": 7}
assert counter.await_args.kwargs["request"].tools == _MALFORMED_TOOL_ENTRY
@ -570,7 +576,7 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set(
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
assert response == {"input_tokens": 7}
counted: Final = counter.await_args.kwargs["request"]
@ -609,12 +615,213 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
patch.object(proxy_server, "llm_router", new=router), # test-quality-ok: module global imported at call time; no injection seam on this route
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
assert response == {"input_tokens": 7}
assert counter.await_args.kwargs["request"].tools is None
_ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages"
_ANTHROPIC_COUNT_TOKENS_URL: Final = "https://api.anthropic.com/v1/messages/count_tokens"
_OPTED_IN: Final = {"token_count_billed_request": True}
_WEATHER_TOOL: Final = {
"name": "get_weather",
"description": "Get the current weather for a city",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
}
_COUNT_BODY: Final = {
"model": "claude-count",
"max_tokens": 64,
"system": [{"type": "text", "text": "Answer in one word."}],
"tools": [_WEATHER_TOOL],
"messages": [{"role": "user", "content": [{"type": "text", "text": "What is the weather in Paris?"}]}],
}
def _count_router(*model_infos: Mapping[str, object]) -> Router:
return Router(
model_list=[
{
"model_name": "claude-count",
"litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"},
"model_info": dict(model_info),
}
for model_info in model_infos
],
num_retries=0,
)
def _count_request(body: Mapping[str, object]) -> Request:
payload: Final = json.dumps(body).encode()
async def receive() -> Message:
return {"type": "http.request", "body": payload, "more_body": False}
return Request(
{
"type": "http",
"method": "POST",
"path": "/v1/messages/count_tokens",
"query_string": b"",
"headers": [(b"content-type", b"application/json")],
},
receive,
)
def _billed_reply(model: str) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "msg_count",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "Sunny"}],
"stop_reason": "max_tokens",
"usage": {
"input_tokens": 1200,
"cache_read_input_tokens": 30,
"cache_creation_input_tokens": 4,
"output_tokens": 1,
},
},
)
@pytest.fixture
def count_proxy(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
@pytest.mark.asyncio
@pytest.mark.usefixtures("count_proxy")
async def test_count_tokens_answers_the_billed_input_of_a_one_token_request_when_every_deployment_opts_in(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
monkeypatch.setattr(proxy_server, "llm_router", _count_router(_OPTED_IN, _OPTED_IN))
fastapi_response: Final = Response()
with respx.mock:
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5"))
count_route: Final = respx.post(_ANTHROPIC_COUNT_TOKENS_URL).mock(
return_value=httpx.Response(200, json={"input_tokens": 9})
)
response: Final = await ep.count_tokens(
fastapi_response, _count_request({**_COUNT_BODY, "stream": True}), UserAPIKeyAuth()
)
assert response == {"input_tokens": 1234, "litellm_count_source": "billed_request"}
assert not count_route.called
sent: Final = json.loads(messages_route.calls.last.request.content)
assert sent["max_tokens"] == 1
assert sent.get("stream", False) is False
assert sent["system"] == _COUNT_BODY["system"]
assert sent["tools"] == _COUNT_BODY["tools"]
assert sent["messages"] == _COUNT_BODY["messages"]
assert float(fastapi_response.headers["x-litellm-response-cost"]) > 0
@pytest.mark.asyncio
@pytest.mark.usefixtures("count_proxy")
@pytest.mark.parametrize(
"model_infos",
[({}, {}), (_OPTED_IN, {}), ({"token_count_billed_request": "true"},)],
ids=["knob_off", "one_of_two_deployments", "string_value"],
)
async def test_count_tokens_keeps_the_provider_count_unless_every_deployment_opts_in(
monkeypatch: pytest.MonkeyPatch, model_infos: tuple[Mapping[str, object], ...]
) -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
monkeypatch.setattr(proxy_server, "llm_router", _count_router(*model_infos))
with respx.mock:
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5"))
respx.post(_ANTHROPIC_COUNT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9}))
response: Final = await ep.count_tokens(Response(), _count_request(_COUNT_BODY), UserAPIKeyAuth())
assert response == {"input_tokens": 9, "litellm_count_source": "provider"}
assert not messages_route.called
@pytest.mark.asyncio
@pytest.mark.usefixtures("count_proxy")
async def test_count_tokens_estimates_a_model_with_no_deployments_without_a_billed_request(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
monkeypatch.setattr(proxy_server, "llm_router", _count_router(_OPTED_IN))
with respx.mock:
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5"))
response: Final = await ep.count_tokens(
Response(), _count_request({**_COUNT_BODY, "model": "claude-unlisted"}), UserAPIKeyAuth()
)
assert response["litellm_count_source"] == "local_estimate"
assert not messages_route.called
@pytest.mark.asyncio
@pytest.mark.usefixtures("count_proxy")
async def test_count_tokens_answers_a_failed_billed_request_as_its_error_without_falling_back(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
router: Final = Router(
model_list=[
{
"model_name": "claude-count",
"litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"},
"model_info": dict(_OPTED_IN),
},
{
"model_name": "claude-backup",
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-test"},
},
],
fallbacks=[{"claude-count": ["claude-backup"]}],
num_retries=0,
)
monkeypatch.setattr(proxy_server, "llm_router", router)
def reply(request: httpx.Request) -> httpx.Response:
model: Final = json.loads(request.content)["model"]
if model == "claude-haiku-5-5":
return _billed_reply(model)
return httpx.Response(
429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "Number of requests is high"}}
)
with respx.mock:
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(side_effect=reply)
response: Final = await ep.count_tokens(Response(), _count_request(_COUNT_BODY), UserAPIKeyAuth())
assert isinstance(response, Response)
assert response.status_code == 429
body: Final = json.loads(response.body)
assert body["type"] == "error"
assert body["error"]["type"] == "rate_limit_error"
assert [json.loads(call.request.content)["model"] for call in messages_route.calls] == ["claude-sonnet-5-5"]
class TestEventLoggingBatchEndpoint:
"""Test the stubbed event logging batch endpoint"""

View file

@ -34,7 +34,7 @@ load_dotenv()
# this file is to test litellm/proxy
from fastapi import Request
from fastapi import Request, Response
from fastapi.responses import JSONResponse
from starlette.types import Message
@ -386,7 +386,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
try:
# Call the endpoint
response = await count_tokens(mock_request, mock_user_api_key_dict)
response = await count_tokens(Response(), mock_request, mock_user_api_key_dict)
# Verify response format matches Anthropic spec
assert isinstance(response, dict)
@ -466,7 +466,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
try:
# Call the endpoint
response = await count_tokens(mock_request, mock_user_api_key_dict)
response = await count_tokens(Response(), mock_request, mock_user_api_key_dict)
# Verify response format matches Anthropic spec
assert isinstance(response, dict)
@ -574,7 +574,7 @@ async def test_anthropic_endpoint_error_handling():
anthropic_endpoints.read_request_body = mock_read_request_body
try:
response = await count_tokens(mock_request, mock_user_api_key_dict)
response = await count_tokens(Response(), mock_request, mock_user_api_key_dict)
assert response.status_code == 400
assert json.loads(response.body) == {
@ -615,6 +615,8 @@ async def test_factory_anthropic_endpoint_calls_anthropic_counter():
}
]
mock_router.get_model_list.return_value = mock_router.model_list
# Mock the async method properly
mock_router.async_get_available_deployment = AsyncMock(
return_value={
@ -676,6 +678,8 @@ async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter():
}
]
mock_router.get_model_list.return_value = mock_router.model_list
# Mock the async method properly
mock_router.async_get_available_deployment = AsyncMock(
return_value={
@ -1273,7 +1277,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
proxy_server.count_request_tokens = mock_token_counter_error
try:
response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict)
response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
@ -1330,7 +1334,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
proxy_server.count_request_tokens = mock_token_counter_error
try:
response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict)
response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict)
assert isinstance(response, JSONResponse)
assert response.status_code == 403
@ -1387,7 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
proxy_server.count_request_tokens = mock_token_counter_error
try:
response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict)
response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict)
assert isinstance(response, JSONResponse)
assert response.status_code == 429
@ -1463,7 +1467,7 @@ async def test_anthropic_count_tokens_drops_a_malformed_tool_result_text_when_dr
route: Final = respx.post("https://api.openai.com/v1/responses/input_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 9})
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
response: Final = await anthropic_count_tokens(Response(), request, UserAPIKeyAuth())
assert response["input_tokens"] == 9
sent: Final = json.loads(route.calls.last.request.content)
@ -1508,7 +1512,7 @@ async def _count_on_a_model_without_a_count_api(
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
with respx.mock:
return await anthropic_count_tokens(request, UserAPIKeyAuth())
return await anthropic_count_tokens(Response(), request, UserAPIKeyAuth())
@pytest.mark.asyncio
@ -1589,7 +1593,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 7})
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
response: Final = await anthropic_count_tokens(Response(), request, UserAPIKeyAuth())
assert response["input_tokens"] == 7
sent: Final = json.loads(route.calls.last.request.content)
@ -3033,7 +3037,7 @@ async def _count_through_the_messages_endpoint(
return_value=AsyncHTTPHandler(transport=httpx.MockTransport(peer)),
),
):
response: Final = await anthropic_count_tokens(mock_request, MagicMock())
response: Final = await anthropic_count_tokens(Response(), mock_request, MagicMock())
return response, peer.request_bodies

View file

@ -47183,7 +47183,7 @@ export interface components {
* Litellm Count Source
* @enum {string}
*/
litellm_count_source?: "provider" | "token_count_model" | "local_estimate";
litellm_count_source?: "provider" | "token_count_model" | "local_estimate" | "billed_request";
/** Prompttokensdetails */
promptTokensDetails: components["schemas"]["PromptTokensDetails"][];
/** Totaltokens */
@ -47213,7 +47213,7 @@ export interface components {
/** TokenCountResponse */
TokenCountResponse: {
/** Count Source */
count_source?: ("provider" | "token_count_model" | "local_estimate") | null;
count_source?: ("provider" | "token_count_model" | "local_estimate" | "billed_request") | null;
/**
* Error
* @default false