diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index b12676e08f3..b16bbac7b09 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,7 @@ from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from pydantic import JsonValue, ValidationError +from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -48,7 +48,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_type, with_litellm_call_id, ) -from litellm.proxy.common_utils.token_count_settings import count_source_fields +from litellm.proxy.common_utils.token_count_settings import count_source_fields, counts_with_billed_request from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli from litellm.router import Router @@ -169,6 +169,20 @@ async def anthropic_response( This was a BETA endpoint that calls 100+ LLMs in the anthropic format. """ + return await _anthropic_messages_response( + ProxyBaseLLMRequestProcessing(data=await read_request_body(request=request)), + fastapi_response, + request, + user_api_key_dict, + ) + + +async def _anthropic_messages_response( + base_llm_response_processor: ProxyBaseLLMRequestProcessing, + fastapi_response: Response, + request: Request, + user_api_key_dict: UserAPIKeyAuth, +) -> object: from litellm.proxy.proxy_server import ( general_settings, llm_router, @@ -182,8 +196,7 @@ async def anthropic_response( version, ) - data: Final = await read_request_body(request=request) - base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data) + data: Final = base_llm_response_processor.data try: result: Final = await base_llm_response_processor.base_process_llm_request( request=request, @@ -244,7 +257,7 @@ async def anthropic_response( usage=_usage, ) - if data.get("stream", None) is not None and data["stream"] is True: + if "stream" in data and data["stream"] is True: # For streaming, use the standard SSE data generator async def _passthrough_stream_generator(): yield _anthropic_response @@ -334,6 +347,43 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue: _COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params") +class _BilledUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: int + cache_read_input_tokens: int | None = None + cache_creation_input_tokens: int | None = None + + @property + def billed_input_tokens(self) -> int: + return self.input_tokens + (self.cache_read_input_tokens or 0) + (self.cache_creation_input_tokens or 0) + + +class _BilledResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + usage: _BilledUsage + + +def _counts_with_billed_request(llm_router: Router | None, model: str, team_id: str | None) -> bool: + if llm_router is None: + return False + return counts_with_billed_request(llm_router.get_model_list(model_name=model, team_id=team_id) or ()) + + +async def _billed_input_tokens( + probe: ProxyBaseLLMRequestProcessing, + fastapi_response: Response, + request: Request, + user_api_key_dict: UserAPIKeyAuth, +) -> object: + response: Final = await _anthropic_messages_response(probe, fastapi_response, request, user_api_key_dict) + if isinstance(response, Response): + return response + billed: Final = _BilledResponse.model_validate(response, from_attributes=True) + return {"input_tokens": billed.usage.billed_input_tokens, "litellm_count_source": "billed_request"} + + class _CountTokensBodyOptions(TypedDict): output_format: ReadOnly[JsonValue] context_management: ReadOnly[JsonValue] @@ -357,6 +407,7 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: dependencies=[Depends(user_api_key_auth)], ) async def count_tokens( + fastapi_response: Response, request: Request, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # Used for auth ): @@ -397,6 +448,18 @@ async def count_tokens( if not messages: raise HTTPException(status_code=400, detail="messages parameter is required") + if isinstance(model_name, str) and _counts_with_billed_request( + llm_router, model_name, user_api_key_dict.team_id + ): + return await _billed_input_tokens( + ProxyBaseLLMRequestProcessing( + data={**request_data, "max_tokens": 1, "stream": False, "disable_fallbacks": True} + ), + fastapi_response, + request, + user_api_key_dict, + ) + auto_drop_params: Final = should_auto_drop_params_for_agentic_cli( request.headers.get("user-agent", ""), "drop_params" in data, proxy_config ) diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 22b35bd1001..7d503dd0d80 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace from types import MappingProxyType from typing import Final, Literal @@ -176,6 +176,13 @@ def resolve_token_count_settings( ) +def counts_with_billed_request(deployments: Sequence[Mapping[str, object]]) -> bool: + return bool(deployments) and all( + (as_str_mapping(deployment.get("model_info")) or {}).get("token_count_billed_request") is True + for deployment in deployments + ) + + def count_source_fields(response: TokenCountResponse) -> TokenCountSourceFields: match response.count_source: case None: diff --git a/litellm/types/token_count.py b/litellm/types/token_count.py index 90ccefea177..50f4f873122 100644 --- a/litellm/types/token_count.py +++ b/litellm/types/token_count.py @@ -2,7 +2,7 @@ from typing import Literal, TypeAlias from typing_extensions import NotRequired, ReadOnly, TypedDict -TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate"] +TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate", "billed_request"] TokenCountTokenizer: TypeAlias = Literal["kimi_k3"] TokenCountProvider: TypeAlias = Literal["bedrock", "bedrock_mantle"] diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index c7bb05e5d30..1b7bb913207 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -5,12 +5,18 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec import json import logging import unittest +from collections.abc import Mapping from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx +from fastapi import Request, Response from fastapi.testclient import TestClient +from starlette.types import Message +import litellm from litellm import Router from litellm._logging import verbose_proxy_logger from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -462,7 +468,7 @@ class TestErrorLogCarriesCallId: ), # test-quality-ok: module global imported at call time; the test targets the endpoint's except block caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"), ): - response = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth()) assert response.status_code == 500 record = self._error_record(caplog) @@ -514,7 +520,7 @@ async def test_count_tokens_answers_400_for_malformed_tools_without_counting(mod patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts it is never reached patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth()) assert response.status_code == 400 assert json.loads(response.body)["error"]["type"] == "invalid_request_error" @@ -537,7 +543,7 @@ async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth()) assert response == {"input_tokens": 7} assert counter.await_args.kwargs["request"].tools == _MALFORMED_TOOL_ENTRY @@ -570,7 +576,7 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set( patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth()) assert response == {"input_tokens": 7} counted: Final = counter.await_args.kwargs["request"] @@ -609,12 +615,213 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives patch.object(proxy_server, "llm_router", new=router), # test-quality-ok: module global imported at call time; no injection seam on this route ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth()) assert response == {"input_tokens": 7} assert counter.await_args.kwargs["request"].tools is None +_ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages" +_ANTHROPIC_COUNT_TOKENS_URL: Final = "https://api.anthropic.com/v1/messages/count_tokens" +_OPTED_IN: Final = {"token_count_billed_request": True} +_WEATHER_TOOL: Final = { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, +} +_COUNT_BODY: Final = { + "model": "claude-count", + "max_tokens": 64, + "system": [{"type": "text", "text": "Answer in one word."}], + "tools": [_WEATHER_TOOL], + "messages": [{"role": "user", "content": [{"type": "text", "text": "What is the weather in Paris?"}]}], +} + + +def _count_router(*model_infos: Mapping[str, object]) -> Router: + return Router( + model_list=[ + { + "model_name": "claude-count", + "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"}, + "model_info": dict(model_info), + } + for model_info in model_infos + ], + num_retries=0, + ) + + +def _count_request(body: Mapping[str, object]) -> Request: + payload: Final = json.dumps(body).encode() + + async def receive() -> Message: + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/messages/count_tokens", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + + +def _billed_reply(model: str) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_count", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "Sunny"}], + "stop_reason": "max_tokens", + "usage": { + "input_tokens": 1200, + "cache_read_input_tokens": 30, + "cache_creation_input_tokens": 4, + "output_tokens": 1, + }, + }, + ) + + +@pytest.fixture +def count_proxy(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("count_proxy") +async def test_count_tokens_answers_the_billed_input_of_a_one_token_request_when_every_deployment_opts_in( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + monkeypatch.setattr(proxy_server, "llm_router", _count_router(_OPTED_IN, _OPTED_IN)) + fastapi_response: Final = Response() + + with respx.mock: + messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5")) + count_route: Final = respx.post(_ANTHROPIC_COUNT_TOKENS_URL).mock( + return_value=httpx.Response(200, json={"input_tokens": 9}) + ) + response: Final = await ep.count_tokens( + fastapi_response, _count_request({**_COUNT_BODY, "stream": True}), UserAPIKeyAuth() + ) + + assert response == {"input_tokens": 1234, "litellm_count_source": "billed_request"} + assert not count_route.called + sent: Final = json.loads(messages_route.calls.last.request.content) + assert sent["max_tokens"] == 1 + assert sent.get("stream", False) is False + assert sent["system"] == _COUNT_BODY["system"] + assert sent["tools"] == _COUNT_BODY["tools"] + assert sent["messages"] == _COUNT_BODY["messages"] + assert float(fastapi_response.headers["x-litellm-response-cost"]) > 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("count_proxy") +@pytest.mark.parametrize( + "model_infos", + [({}, {}), (_OPTED_IN, {}), ({"token_count_billed_request": "true"},)], + ids=["knob_off", "one_of_two_deployments", "string_value"], +) +async def test_count_tokens_keeps_the_provider_count_unless_every_deployment_opts_in( + monkeypatch: pytest.MonkeyPatch, model_infos: tuple[Mapping[str, object], ...] +) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + monkeypatch.setattr(proxy_server, "llm_router", _count_router(*model_infos)) + + with respx.mock: + messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5")) + respx.post(_ANTHROPIC_COUNT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + response: Final = await ep.count_tokens(Response(), _count_request(_COUNT_BODY), UserAPIKeyAuth()) + + assert response == {"input_tokens": 9, "litellm_count_source": "provider"} + assert not messages_route.called + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("count_proxy") +async def test_count_tokens_estimates_a_model_with_no_deployments_without_a_billed_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + monkeypatch.setattr(proxy_server, "llm_router", _count_router(_OPTED_IN)) + + with respx.mock: + messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5")) + response: Final = await ep.count_tokens( + Response(), _count_request({**_COUNT_BODY, "model": "claude-unlisted"}), UserAPIKeyAuth() + ) + + assert response["litellm_count_source"] == "local_estimate" + assert not messages_route.called + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("count_proxy") +async def test_count_tokens_answers_a_failed_billed_request_as_its_error_without_falling_back( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + router: Final = Router( + model_list=[ + { + "model_name": "claude-count", + "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"}, + "model_info": dict(_OPTED_IN), + }, + { + "model_name": "claude-backup", + "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-test"}, + }, + ], + fallbacks=[{"claude-count": ["claude-backup"]}], + num_retries=0, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + def reply(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model == "claude-haiku-5-5": + return _billed_reply(model) + return httpx.Response( + 429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "Number of requests is high"}} + ) + + with respx.mock: + messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(side_effect=reply) + response: Final = await ep.count_tokens(Response(), _count_request(_COUNT_BODY), UserAPIKeyAuth()) + + assert isinstance(response, Response) + assert response.status_code == 429 + body: Final = json.loads(response.body) + assert body["type"] == "error" + assert body["error"]["type"] == "rate_limit_error" + assert [json.loads(call.request.content)["model"] for call in messages_route.calls] == ["claude-sonnet-5-5"] + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..6c056ff8aba 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -34,7 +34,7 @@ load_dotenv() # this file is to test litellm/proxy -from fastapi import Request +from fastapi import Request, Response from fastapi.responses import JSONResponse from starlette.types import Message @@ -386,7 +386,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): try: # Call the endpoint - response = await count_tokens(mock_request, mock_user_api_key_dict) + response = await count_tokens(Response(), mock_request, mock_user_api_key_dict) # Verify response format matches Anthropic spec assert isinstance(response, dict) @@ -466,7 +466,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): try: # Call the endpoint - response = await count_tokens(mock_request, mock_user_api_key_dict) + response = await count_tokens(Response(), mock_request, mock_user_api_key_dict) # Verify response format matches Anthropic spec assert isinstance(response, dict) @@ -574,7 +574,7 @@ async def test_anthropic_endpoint_error_handling(): anthropic_endpoints.read_request_body = mock_read_request_body try: - response = await count_tokens(mock_request, mock_user_api_key_dict) + response = await count_tokens(Response(), mock_request, mock_user_api_key_dict) assert response.status_code == 400 assert json.loads(response.body) == { @@ -615,6 +615,8 @@ async def test_factory_anthropic_endpoint_calls_anthropic_counter(): } ] + mock_router.get_model_list.return_value = mock_router.model_list + # Mock the async method properly mock_router.async_get_available_deployment = AsyncMock( return_value={ @@ -676,6 +678,8 @@ async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter(): } ] + mock_router.get_model_list.return_value = mock_router.model_list + # Mock the async method properly mock_router.async_get_available_deployment = AsyncMock( return_value={ @@ -1273,7 +1277,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): proxy_server.count_request_tokens = mock_token_counter_error try: - response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict) assert isinstance(response, JSONResponse) assert response.status_code == 400 @@ -1330,7 +1334,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): proxy_server.count_request_tokens = mock_token_counter_error try: - response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict) assert isinstance(response, JSONResponse) assert response.status_code == 403 @@ -1387,7 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): proxy_server.count_request_tokens = mock_token_counter_error try: - response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict) assert isinstance(response, JSONResponse) assert response.status_code == 429 @@ -1463,7 +1467,7 @@ async def test_anthropic_count_tokens_drops_a_malformed_tool_result_text_when_dr route: Final = respx.post("https://api.openai.com/v1/responses/input_tokens").mock( return_value=httpx.Response(200, json={"input_tokens": 9}) ) - response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + response: Final = await anthropic_count_tokens(Response(), request, UserAPIKeyAuth()) assert response["input_tokens"] == 9 sent: Final = json.loads(route.calls.last.request.content) @@ -1508,7 +1512,7 @@ async def _count_on_a_model_without_a_count_api( monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) with respx.mock: - return await anthropic_count_tokens(request, UserAPIKeyAuth()) + return await anthropic_count_tokens(Response(), request, UserAPIKeyAuth()) @pytest.mark.asyncio @@ -1589,7 +1593,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( return_value=httpx.Response(200, json={"input_tokens": 7}) ) - response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + response: Final = await anthropic_count_tokens(Response(), request, UserAPIKeyAuth()) assert response["input_tokens"] == 7 sent: Final = json.loads(route.calls.last.request.content) @@ -3033,7 +3037,7 @@ async def _count_through_the_messages_endpoint( return_value=AsyncHTTPHandler(transport=httpx.MockTransport(peer)), ), ): - response: Final = await anthropic_count_tokens(mock_request, MagicMock()) + response: Final = await anthropic_count_tokens(Response(), mock_request, MagicMock()) return response, peer.request_bodies diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0fc0d86b479..038170aff2e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -47183,7 +47183,7 @@ export interface components { * Litellm Count Source * @enum {string} */ - litellm_count_source?: "provider" | "token_count_model" | "local_estimate"; + litellm_count_source?: "provider" | "token_count_model" | "local_estimate" | "billed_request"; /** Prompttokensdetails */ promptTokensDetails: components["schemas"]["PromptTokensDetails"][]; /** Totaltokens */ @@ -47213,7 +47213,7 @@ export interface components { /** TokenCountResponse */ TokenCountResponse: { /** Count Source */ - count_source?: ("provider" | "token_count_model" | "local_estimate") | null; + count_source?: ("provider" | "token_count_model" | "local_estimate" | "billed_request") | null; /** * Error * @default false