diff --git a/litellm/_logging.py b/litellm/_logging.py index d08254db3fd..6bfca44aaa8 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -9,6 +9,7 @@ import sys from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter +from types import MappingProxyType from typing import Final, TextIO from urllib.parse import unquote @@ -755,17 +756,20 @@ class JsonFormatter(Formatter): # fields, and a message that merely happens to parse as JSON/dict (e.g. a proxy # log line dumping raw request headers) must never be able to claim them, even on # a record the filter hasn't stamped yet (no correlation context active for it). + extras: Final = MappingProxyType( + {key: value for key, value in _record_attributes(record).items() if key not in _NON_EXTRA_RECORD_ATTRS} + ) parsed = _try_parse_json_message(message_str) if parsed is None: parsed = _try_parse_embedded_python_dict(message_str) if parsed is not None: for key, value in parsed.items(): - if key not in json_record and key not in _RESERVED_CORRELATION_FIELDS: + if key not in json_record and key not in _RESERVED_CORRELATION_FIELDS and key not in extras: json_record[key] = value # Include extra attributes passed via logger.debug("msg", extra={...}) - for key, value in _record_attributes(record).items(): - if key not in _NON_EXTRA_RECORD_ATTRS and key not in json_record: + for key, value in extras.items(): + if key not in json_record: json_record[key] = value # trace_id/session_id are reserved: CorrelationContextFilter is the only diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index a9f95c1f9b9..bf9c119f6f2 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -206,6 +206,10 @@ class BedrockCountTokensConfig(BaseAWSLLM): encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode() return {"input": {"invokeModel": {"body": encoded_body}}} + @staticmethod + def count_tokens_model_id(model: str) -> str: + return get_bedrock_base_model(model).removeprefix("bedrock/") + def get_bedrock_count_tokens_endpoint( self, model: str, @@ -225,12 +229,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): Returns: Complete endpoint URL for CountTokens API """ - # Use existing LiteLLM function to get the base model ID (removes region prefix) - model_id = get_bedrock_base_model(model) - - # Remove bedrock/ prefix if present - model_id = model_id.removeprefix("bedrock/") # Remove "bedrock/" prefix - encoded_model_id: Final = self.encode_model_id(model_id=model_id) + encoded_model_id: Final = self.encode_model_id(model_id=self.count_tokens_model_id(model)) base_url, _ = self.get_runtime_endpoint( api_base=api_base, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 49d23d82aa9..8c3458de928 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3461,12 +3461,12 @@ class LiteLLM_SpendLogs_ResponseObject(LiteLLMPydanticObjectBase): class TokenCountRequest(LiteLLMPydanticObjectBase): model: str prompt: str | None = None - messages: list[dict] | None = None + messages: list[dict[Any, Any]] | None = None """ Anthropic token counting endpoint uses /messages """ - contents: list[dict] | None = None + contents: list[dict[Any, Any]] | None = None """ Google /countTokens endpoint expects contents to be a list of dicts with the following structure: """ diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index b12676e08f3..3e01bb01576 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -411,6 +411,7 @@ async def count_tokens( request=token_request, call_endpoint=True, messages_format="anthropic", + litellm_call_id=litellm_call_id, **_count_tokens_body_options(data, auto_drop_params), ) except HTTPException as e: diff --git a/litellm/proxy/common_utils/token_count_log.py b/litellm/proxy/common_utils/token_count_log.py new file mode 100644 index 00000000000..3d92fa31ada --- /dev/null +++ b/litellm/proxy/common_utils/token_count_log.py @@ -0,0 +1,116 @@ +import json +import re +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, TypeVar + +from fastapi import status + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.openai_error_payload import error_status_code +from litellm.types.token_count import TokenCountSource +from litellm.types.utils import TokenCountResponse + +_BARE_LOGFMT_VALUE: Final = re.compile(r"[\w.:/@+-]+", re.ASCII) +_JSON_ESCAPED_BRACES: Final = str.maketrans({"{": "\\u007b", "}": "\\u007d"}) + + +def _logfmt_value(value: str | None) -> str: + if value is None: + return "None" + if _BARE_LOGFMT_VALUE.fullmatch(value): + return value + return json.dumps(value).translate(_JSON_ESCAPED_BRACES) + + +_Counted: Final = TypeVar("_Counted") +_ProviderCount: Final = TypeVar("_ProviderCount", bound=Mapping[str, object]) + + +@dataclass(frozen=True, slots=True) +class _CountFields: + model_used: str | None + input_tokens: int | None + count_source: TokenCountSource | None + + +_FAILED_COUNT: Final = _CountFields(model_used=None, input_tokens=None, count_source=None) + + +@dataclass(frozen=True, slots=True) +class _CountRequest: + litellm_call_id: str + request_model: str + model_id: str | None + + +def deployment_model_id(deployment: Mapping[str, object] | None) -> str | None: + match deployment: + case {"model_info": {"id": str() as model_id}}: + return model_id + case _: + return None + + +def _log_token_count(request: _CountRequest, fields: _CountFields, status_code: int) -> None: + verbose_proxy_logger.info( + "token count - litellm_call_id=%s request_model=%s model_id=%s model_used=%s input_tokens=%s " + "count_source=%s status_code=%s", + _logfmt_value(request.litellm_call_id), + _logfmt_value(request.request_model), + _logfmt_value(request.model_id), + _logfmt_value(fields.model_used), + fields.input_tokens, + fields.count_source, + status_code, + extra=MappingProxyType( + { + "litellm_call_id": request.litellm_call_id, + "request_model": request.request_model, + "model_id": request.model_id, + "model_used": fields.model_used, + "input_tokens": fields.input_tokens, + "count_source": fields.count_source, + "status_code": status_code, + } + ), + ) + + +async def _logged( + count: Awaitable[_Counted], request: _CountRequest, fields_of: Callable[[_Counted], _CountFields] +) -> _Counted: + try: + counted: Final = await count + except Exception as e: + _log_token_count(request, _FAILED_COUNT, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR)) + raise + _log_token_count(request, fields_of(counted), status.HTTP_200_OK) + return counted + + +def _token_count_response_fields(counted: TokenCountResponse) -> _CountFields: + return _CountFields( + model_used=counted.model_used, input_tokens=counted.total_tokens, count_source=counted.count_source + ) + + +async def logged_token_count( + count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str, model_id: str | None +) -> TokenCountResponse: + return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), _token_count_response_fields) + + +async def logged_provider_token_count( + count: Awaitable[_ProviderCount], litellm_call_id: str, request_model: str, model_id: str | None, model_used: str +) -> _ProviderCount: + def provider_fields(counted: _ProviderCount) -> _CountFields: + input_tokens: Final = counted.get("input_tokens") + return _CountFields( + model_used=model_used, + input_tokens=input_tokens if isinstance(input_tokens, int) and not isinstance(input_tokens, bool) else None, + count_source="provider", + ) + + return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), provider_fields) diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 3d1a9ab4cfe..6a3f1d5a197 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -21,6 +21,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export read_request_body, ) +from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER from litellm.proxy.common_utils.token_count_settings import count_source_fields from litellm.types.llms.vertex_ai import TokenCountDetailsResponse @@ -286,6 +287,7 @@ async def google_count_tokens(request: Request, model_name: str): from litellm.proxy.common_utils.http_parsing_utils import read_request_body from litellm.proxy.proxy_server import token_counter as internal_token_counter + litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)) data: Final = await read_request_body(request=request) try: token_request: Final = _google_count_tokens_request(model_name, data) @@ -298,13 +300,15 @@ async def google_count_tokens(request: Request, model_name: str): ) try: - token_response: Final = await internal_token_counter(request=token_request, call_endpoint=True) + token_response: Final = await internal_token_counter( + request=token_request, call_endpoint=True, litellm_call_id=litellm_call_id + ) except ProxyException as error: return _google_error_response(request, error) except HTTPException as error: return _google_error_response(request, proxy_exception_from_http_exception(error, {})) except Exception as error: - log_llm_api_exception(error, resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))) + log_llm_api_exception(error, litellm_call_id) return _google_error_response( request, ProxyException(message="Internal server error", type="internal_server_error", param=None, code=500), diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 9eef4f0ef9d..a9e908d2a58 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -87,6 +87,7 @@ from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, model_deployments_support_stream_options, open_sse_before_first_byte, + resolve_litellm_call_id, ) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -99,10 +100,12 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg safe_get_request_headers, safe_set_request_parsed_body, ) +from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) +from litellm.proxy.common_utils.token_count_log import deployment_model_id, logged_provider_token_count from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( stream_usage_relay, @@ -1272,34 +1275,35 @@ async def handle_bedrock_count_tokens( litellm_params: Final = {"user_api_key_dict": user_api_key_dict} resolved_model = model # Default fallback - if llm_router: - deployments: Final = llm_router.get_model_list(model_name=model) - if deployments and len(deployments) > 0: - # Get the first matching deployment - deployment: Final = deployments[0] - model_litellm_params: Final = deployment.get("litellm_params", {}) + deployments: Final = llm_router.get_model_list(model_name=model) if llm_router else None + deployment: Final = deployments[0] if deployments else None + if deployment is not None: + model_litellm_params: Final = deployment.get("litellm_params", {}) - # Get the resolved model ID from the configuration - if "model" in model_litellm_params: - resolved_model = model_litellm_params["model"] + # Get the resolved model ID from the configuration + if "model" in model_litellm_params: + resolved_model = model_litellm_params["model"] - # Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery - for key, value in model_litellm_params.items(): - if key != "user_api_key_dict": # Don't overwrite user_api_key_dict - litellm_params[key] = value + # Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery + for key, value in model_litellm_params.items(): + if key != "user_api_key_dict": # Don't overwrite user_api_key_dict + litellm_params[key] = value verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params) verbose_proxy_logger.debug("Resolved model: %s", resolved_model) - # Handle the count tokens request - result: Final = await handler.handle_count_tokens_request( - request_data=request_body, - litellm_params=litellm_params, - resolved_model=resolved_model, + return await logged_provider_token_count( + handler.handle_count_tokens_request( + request_data=request_body, + litellm_params=litellm_params, + resolved_model=resolved_model, + ), + litellm_call_id=resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)), + request_model=model, + model_id=deployment_model_id(deployment), + model_used=handler.count_tokens_model_id(resolved_model), ) - return result - except BedrockError as e: # Convert BedrockError to HTTPException for FastAPI verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ca0f8892b9d..3f54d915da3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -552,6 +552,7 @@ from litellm.proxy.common_utils.timezone_utils import ( get_budget_reset_settings, get_budget_reset_time, ) +from litellm.proxy.common_utils.token_count_log import deployment_model_id, logged_token_count from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, end_user_cache_key, @@ -15244,16 +15245,21 @@ def _system_message(system: object) -> ChatCompletionSystemMessage | None: dependencies=[Depends(user_api_key_auth)], response_model=TokenCountResponse, ) -async def token_counter(request: TokenCountRequest, call_endpoint: bool = False): +async def token_counter( + request: TokenCountRequest, + call_endpoint: bool = False, + litellm_call_id: Annotated[str | None, Header(alias=LITELLM_CALL_ID_HEADER)] = None, +): """ Args: request: TokenCountRequest call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs. + litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs Returns: TokenCountResponse """ - return await count_request_tokens(request=request, call_endpoint=call_endpoint) + return await count_request_tokens(request=request, call_endpoint=call_endpoint, litellm_call_id=litellm_call_id) async def count_request_tokens( @@ -15264,6 +15270,53 @@ async def count_request_tokens( context_management: JsonValue = None, request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}), responses_request: Mapping[str, object] | None = None, + litellm_call_id: str | None = None, +) -> TokenCountResponse: + if request.prompt is None and request.messages is None and request.contents is None: + raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided") + routed: Final = await _routed_count_deployment(request.model, responses_request) + return await logged_token_count( + _count_request_tokens( + request=request, + routed=routed, + call_endpoint=call_endpoint, + messages_format=messages_format, + output_format=output_format, + context_management=context_management, + request_litellm_params=request_litellm_params, + responses_request=responses_request, + ), + litellm_call_id=resolve_litellm_call_id(litellm_call_id), + request_model=request.model, + model_id=deployment_model_id(routed), + ) + + +async def _routed_count_deployment( + model: str, responses_request: Mapping[str, object] | None +) -> Mapping[str, object] | None: + if llm_router is None: + return None + try: + return await llm_router.async_get_available_deployment( + model=model, request_kwargs=dict(_responses_routing_kwargs(responses_request)) + ) + except Exception: + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment" + ) + return None + + +async def _count_request_tokens( + request: TokenCountRequest, + routed: Mapping[str, object] | None, + call_endpoint: bool, + messages_format: TokenCountMessagesFormat, + output_format: JsonValue, + context_management: JsonValue, + request_litellm_params: Mapping[str, JsonValue], + responses_request: Mapping[str, object] | None, ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -15271,26 +15324,9 @@ async def count_request_tokens( tools: Final = request.tools system: Final = request.system - ######################################################### - # Validate request - ######################################################### - if prompt is None and messages is None and contents is None: - raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided") - - deployment: dict[str, Any] | None = None + deployment: dict[str, Any] | None = None if routed is None else dict(routed) litellm_model_name = None model_info: ModelMapInfo | None = None - if llm_router is not None: - # get 1 deployment corresponding to the model - try: - deployment = await llm_router.async_get_available_deployment( - model=request.model, - request_kwargs=dict(_responses_routing_kwargs(responses_request)), - ) - except Exception: - verbose_proxy_logger.exception( - "litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment" - ) if deployment is not None: litellm_model_name = deployment.get("litellm_params", {}).get("model") model_info = deployment.get("model_info", {}) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 312c5124ca5..fc1513b84d8 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1407,7 +1407,12 @@ class _TokenCountPayload(TypedDict): class _TokenCounter(Protocol): def __call__( - self, *, request: TokenCountRequest, call_endpoint: bool, responses_request: Mapping[str, object] | None + self, + *, + request: TokenCountRequest, + call_endpoint: bool, + responses_request: Mapping[str, object] | None, + litellm_call_id: str | None, ) -> Awaitable[TokenCountResponse]: ... @@ -1700,7 +1705,10 @@ async def responses_input_tokens( ) try: token_response: Final = await token_counter( - request=token_request, call_endpoint=True, responses_request=responses_request + request=token_request, + call_endpoint=True, + responses_request=responses_request, + litellm_call_id=request.headers.get(LITELLM_CALL_ID_HEADER), ) except ProxyException as e: if e.type != _PROVIDER_COUNT_ERROR_TYPE: diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index c7bb05e5d30..c041ebe56a8 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -5,9 +5,11 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec import json import logging import unittest +from collections.abc import Callable, Mapping from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi.testclient import TestClient @@ -615,6 +617,104 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e assert counter.await_args.kwargs["request"].tools is None +def _count_on_claude( + monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str] +) -> httpx.Response: + from fastapi import FastAPI + + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.anthropic_endpoints.endpoints import router + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"} + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "count-log-claude", + "litellm_params": {**claude, "order": 2}, + "model_info": {"id": "count-log-fallback-deployment"}, + }, + { + "model_name": "count-log-claude", + "litellm_params": {**claude, "order": 1}, + "model_info": {"id": "count-log-routed-deployment"}, + }, + ] + ), + ) + monkeypatch.setattr( + anthropic_count_tokens_handler, + "get_async_httpx_client", + lambda *args, **kwargs: AsyncHTTPHandler(transport=httpx.MockTransport(peer)), + ) + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth() + return TestClient(app).post( + "/v1/messages/count_tokens", + json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]}, + headers=dict(headers), + ) + + +def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response(200, json={"input_tokens": 77}, request=request), + {"x-litellm-call-id": "count-log-msg-claude"}, + ) + + assert response.json() == {"input_tokens": 77, "litellm_count_source": "provider"}, response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-msg-claude", + "request_model": "count-log-claude", + "model_id": "count-log-routed-deployment", + "model_used": "claude-zeta-9", + "input_tokens": 77, + "count_source": "provider", + "status_code": 200, + }, + ) + + +def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + rejection: Final = { + "type": "error", + "error": {"type": "invalid_request_error", "message": "messages.0.content.0: invalid base64 data"}, + } + + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response(400, json=rejection, request=request), + {"x-litellm-call-id": "count-log-msg-error"}, + ) + + assert response.status_code == 400, response.text + assert response.json()["error"] == rejection["error"] + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-msg-error", + "request_model": "count-log-claude", + "model_id": "count-log-routed-deployment", + "model_used": None, + "input_tokens": None, + "count_source": None, + "status_code": 400, + }, + ) + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 58c9a5e4d76..e13eddef43b 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -3,6 +3,7 @@ import asyncio import copy import inspect +import logging import os import tempfile import warnings @@ -17,6 +18,7 @@ from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server +from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceTypes from litellm.integrations.otel.model.payloads import ServiceSpanData from litellm.integrations.otel.model.spans import service_span_name @@ -424,6 +426,31 @@ def fresh_agent_read_through(monkeypatch): return read_through +TOKEN_COUNT_LOG_FIELDS: Final = ( + "litellm_call_id", + "request_model", + "model_id", + "model_used", + "input_tokens", + "count_source", + "status_code", +) + + +@pytest.fixture +def token_count_log_lines( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> Iterator[Callable[[], tuple[dict[str, object], ...]]]: + """The structured fields of every INFO token count line the proxy logger writes while the test runs.""" + monkeypatch.setattr(verbose_proxy_logger, "propagate", True) + with caplog.at_level(logging.INFO, logger=verbose_proxy_logger.name): + yield lambda: tuple( + {field: record.__dict__[field] for field in TOKEN_COUNT_LOG_FIELDS} + for record in caplog.records + if record.levelno == logging.INFO and "count_source" in record.__dict__ + ) + + @pytest.fixture def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]: """The ``postgres.{verb} {table}`` names OTel would render for every DB service event diff --git a/tests/unit/proxy/google_endpoints/test_endpoints.py b/tests/unit/proxy/google_endpoints/test_endpoints.py index 17af52b5579..80e45edf897 100644 --- a/tests/unit/proxy/google_endpoints/test_endpoints.py +++ b/tests/unit/proxy/google_endpoints/test_endpoints.py @@ -94,6 +94,7 @@ async def test_proxy_gemini_to_openai_like_model_token_counting(): request=Request( scope={ "type": "http", + "headers": [], "parsed_body": ( ["contents"], {"contents": [{"parts": [{"text": "Hello, how are you?"}]}]}, diff --git a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py index 2777d18a399..bd6315ce481 100644 --- a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py +++ b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py @@ -3,9 +3,11 @@ Test to verify the Google GenAI proxy API endpoints """ +from collections.abc import Callable from typing import Final from unittest.mock import AsyncMock, patch +import httpx import pytest from litellm.types.utils import TokenCountResponse @@ -240,3 +242,56 @@ def test_google_count_tokens_uses_normalized_total_when_provider_shape_differs() assert body["promptTokensDetails"] == [] assert body["litellm_count_source"] == "token_count_model" assert body["litellm_count_model"] == "us.anthropic.claude-opus-5-5" + + +@pytest.mark.parametrize("path", ["/v1beta/models/count-log-claude:countTokens", "/models/count-log-claude:countTokens"]) +def test_google_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], path: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm import Router + from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"} + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "count-log-claude", + "litellm_params": claude, + "model_info": {"id": "count-log-gemini-deployment"}, + } + ] + ), + ) + monkeypatch.setattr( + anthropic_count_tokens_handler, + "get_async_httpx_client", + lambda *args, **kwargs: AsyncHTTPHandler( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"input_tokens": 77}, request=request) + ) + ), + ) + + response: Final = _build_test_client().post( + path, + json={"contents": [{"role": "user", "parts": [{"text": "Hello"}]}]}, + headers={"x-litellm-call-id": "count-log-gemini"}, + ) + + assert (response.status_code, response.json()["totalTokens"]) == (200, 77), response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-gemini", + "request_model": "count-log-claude", + "model_id": "count-log-gemini-deployment", + "model_used": "claude-zeta-9", + "input_tokens": 77, + "count_source": "provider", + "status_code": 200, + }, + ) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 60043daa402..a859c64da07 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2142,7 +2142,6 @@ class TestBedrockPassthroughAwsErrors: assert exc_info.value.aws_error_type == "ValidationException" assert (exc_info.value.error.message, exc_info.value.error.code) == (message, status_code) - @pytest.mark.asyncio async def test_bedrock_request_check_is_raised_with_the_aws_error_type_it_carries( self, monkeypatch: pytest.MonkeyPatch @@ -2183,6 +2182,7 @@ class TestBedrockPassthroughAwsErrors: await self._answer(error) assert (exc_info.value.code, exc_info.value.headers.get("x-model-group")) == ("403", "modèle") + @pytest.mark.asyncio @pytest.mark.parametrize( ("error", "message", "code"), @@ -7250,6 +7250,107 @@ async def test_bedrock_count_tokens_error_forwards_provider_headers(): assert exc_info.value.headers["llm_provider-x-amzn-requestid"] == "req-count-tokens-500" +_BEDROCK_COUNT_ALIAS: Final = "count-log-bedrock" +_BEDROCK_COUNT_MODEL: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +async def _bedrock_native_count( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, aws_reply: httpx.Response +) -> dict[str, object]: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + handle_bedrock_count_tokens, + ) + + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + litellm.Router( + model_list=[ + { + "model_name": _BEDROCK_COUNT_ALIAS, + "litellm_params": { + "model": _BEDROCK_COUNT_MODEL, + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + }, + "model_info": {"id": "count-log-bedrock-deployment"}, + } + ] + ), + ) + respx_mock.post(host="bedrock-runtime.us-east-1.amazonaws.com", path__regex=r"/count-tokens$").mock( + return_value=aws_reply + ) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": f"/bedrock/model/{_BEDROCK_COUNT_ALIAS}/count-tokens", + "headers": [(b"x-litellm-call-id", b"bedrock-native-count")], + } + ) + return await handle_bedrock_count_tokens( + endpoint=f"model/{_BEDROCK_COUNT_ALIAS}/count-tokens", + request=request, + fastapi_response=Response(), + user_api_key_dict=MagicMock(), + request_body={"model": _BEDROCK_COUNT_ALIAS, "messages": [{"role": "user", "content": "hi"}]}, + ) + + +@pytest.mark.asyncio +async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_model( + monkeypatch: pytest.MonkeyPatch, + httpx_transport: None, + respx_mock: respx.MockRouter, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + result: Final = await _bedrock_native_count( + monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12}) + ) + + assert result == {"input_tokens": 12} + assert len(respx_mock.calls) == 1 + assert respx_mock.calls.last.request.url.path == "/model/anthropic.claude-haiku-4-5-20251001-v1:0/count-tokens" + assert token_count_log_lines() == ( + { + "litellm_call_id": "bedrock-native-count", + "request_model": _BEDROCK_COUNT_ALIAS, + "model_id": "count-log-bedrock-deployment", + "model_used": "anthropic.claude-haiku-4-5-20251001-v1:0", + "input_tokens": 12, + "count_source": "provider", + "status_code": 200, + }, + ) + + +@pytest.mark.asyncio +async def test_bedrock_native_count_tokens_logs_the_provider_status_when_the_count_fails( + monkeypatch: pytest.MonkeyPatch, + httpx_transport: None, + respx_mock: respx.MockRouter, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + rejected: Final = httpx.Response(400, json={"message": "The provided model doesn't support counting tokens."}) + with pytest.raises(HTTPException) as exc_info: + await _bedrock_native_count(monkeypatch, respx_mock, rejected) + + assert exc_info.value.status_code == 400 + assert token_count_log_lines() == ( + { + "litellm_call_id": "bedrock-native-count", + "request_model": _BEDROCK_COUNT_ALIAS, + "model_id": "count-log-bedrock-deployment", + "model_used": None, + "input_tokens": None, + "count_source": None, + "status_code": 400, + }, + ) + + class _AzureGroupRouter: def __init__(self, captured: list[dict]) -> None: self.captured = captured diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 985aee061bc..d1af549934a 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -7,7 +7,7 @@ import contextlib import dataclasses import json import unittest -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -19,6 +19,7 @@ from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.constants import EMPTY_MAPPING from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse @@ -4275,6 +4276,7 @@ class TestResponsesInputTokens: llm_router: litellm.Router | None = None, response_id_security: object | None = None, managed_files: object | None = None, + headers: Mapping[str, str] = EMPTY_MAPPING, ) -> tuple[Response, AsyncMock]: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -4307,7 +4309,7 @@ class TestResponsesInputTokens: app.dependency_overrides[_proxy_managed_files_hook] = lambda: managed_files try: client = TestClient(app) - response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876"}) + response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876", **headers}) return response, token_counter_mock finally: app.dependency_overrides.pop(user_api_key_auth, None) @@ -4552,6 +4554,63 @@ class TestResponsesInputTokens: assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text assert sent == [{"model": "gpt-4o-mini", "input": "hi"}] + @pytest.mark.parametrize( + "path", ["/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"] + ) + def test_count_logs_one_line_with_the_callers_call_id_model_and_count( + self, + monkeypatch: pytest.MonkeyPatch, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], + path: str, + ) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http + from litellm.proxy.proxy_server import count_request_tokens + + gpt: Final = {"model": "openai/gpt-4o-mini", "api_key": "test-key", "api_base": "https://count.test/v1"} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "count-log-gpt", + "litellm_params": gpt, + "model_info": {"id": "count-log-responses-deployment"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + monkeypatch.setattr( + openai_count_tokens_http, + "get_async_httpx_client", + lambda *args, **kwargs: AsyncHTTPHandler( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, json={"object": "response.input_tokens", "input_tokens": 7}, request=request + ) + ) + ), + ) + + response, _ = self._post_input_tokens( + {"model": "count-log-gpt", "input": "hi"}, + path=path, + counter=AsyncMock(side_effect=count_request_tokens), + llm_router=router, + headers={"x-litellm-call-id": "count-log-responses"}, + ) + + assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-responses", + "request_model": "count-log-gpt", + "model_id": "count-log-responses-deployment", + "model_used": "gpt-4o-mini", + "input_tokens": 7, + "count_source": "provider", + "status_code": 200, + }, + ) + def test_a_managed_file_the_key_cannot_read_answers_403_without_a_count(self) -> None: from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..fa959d86510 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -10,6 +10,7 @@ import json import logging import threading import traceback +import uuid from collections.abc import Callable, Coroutine, Mapping from functools import partial from pathlib import Path @@ -40,7 +41,7 @@ from starlette.types import Message import litellm from litellm import Router -from litellm._logging import verbose_proxy_logger +from litellm._logging import JsonFormatter, verbose_proxy_logger from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.common_utils import BedrockError @@ -364,6 +365,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -444,6 +446,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -1262,6 +1265,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1319,6 +1323,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1376,6 +1381,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): raise ProxyException( message="Rate limit exceeded", @@ -3892,3 +3898,139 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r ] assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3 spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form")) + + +_COUNTED_HELLO: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"model": "count-log-gpt", "messages": [{"role": "user", "content": "Hello, how many tokens is this?"}]} +) + + +def _utils_token_counter( + monkeypatch: pytest.MonkeyPatch, body: Mapping[str, JsonValue], headers: Mapping[str, str] +) -> httpx.Response: + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "count-log-gpt", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + "model_info": {"id": "count-log-utils-deployment"}, + } + ] + ), + ) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-test")) + return TestClient(app).post("/utils/token_counter", json=dict(body), headers=dict(headers)) + + +def test_utils_token_counter_logs_one_line_with_the_callers_call_id_model_and_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter(monkeypatch, _COUNTED_HELLO, {"x-litellm-call-id": "count-log-utils"}) + + assert response.status_code == 200, response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-utils", + "request_model": "count-log-gpt", + "model_id": "count-log-utils-deployment", + "model_used": "gpt-4o-mini", + "input_tokens": response.json()["total_tokens"], + "count_source": "local_estimate", + "status_code": 200, + }, + ) + + +def test_utils_token_counter_logs_a_generated_call_id_when_the_client_sends_none( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter(monkeypatch, _COUNTED_HELLO, {}) + + assert response.status_code == 200, response.text + (line,) = token_count_log_lines() + assert str(uuid.UUID(str(line["litellm_call_id"]))) == line["litellm_call_id"] + + +def test_utils_token_counter_failed_count_logs_its_status_in_place_of_a_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter( + monkeypatch, {**_COUNTED_HELLO, "prompt": "and a prompt"}, {"x-litellm-call-id": "count-log-utils-error"} + ) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "prompt cannot be sent with messages or contents. Set drop_params to count them without the prompt", + "type": "invalid_request_error", + "param": "prompt", + "code": "400", + } + } + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-utils-error", + "request_model": "count-log-gpt", + "model_id": "count-log-utils-deployment", + "model_used": None, + "input_tokens": None, + "count_source": None, + "status_code": 400, + }, + ) + + +def test_utils_token_counter_request_with_nothing_to_count_writes_no_line( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter( + monkeypatch, {"model": "count-log-gpt"}, {"x-litellm-call-id": "count-log-utils-empty"} + ) + + assert response.status_code == 400, response.text + assert token_count_log_lines() == () + + +def test_utils_token_counter_json_line_keeps_its_own_fields_when_the_call_id_is_a_dict( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + forged_call_id: Final = "{'component':'forged-comp','logger':'forged.py:1','injected_key':'injected'}" + + _utils_token_counter(monkeypatch, _COUNTED_HELLO, {"x-litellm-call-id": forged_call_id}) + + (line,) = token_count_log_lines() + (record,) = (record for record in caplog.records if "count_source" in record.__dict__) + rendered: Final = json.loads(JsonFormatter().format(record)) + assert line["litellm_call_id"] == forged_call_id + assert {key: rendered.get(key) for key in ("litellm_call_id", "component", "logger", "injected_key")} == { + "litellm_call_id": forged_call_id, + "component": verbose_proxy_logger.name, + "logger": f"{record.filename}:{record.lineno}", + "injected_key": None, + } + + +def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newline( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + forged_model: Final = "count-log-gpt\ntoken count - litellm_call_id=forged input_tokens=1" + + _utils_token_counter(monkeypatch, {**_COUNTED_HELLO, "model": forged_model}, {"x-litellm-call-id": "count log"}) + + (line,) = token_count_log_lines() + (message,) = (record.getMessage() for record in caplog.records if "count_source" in record.__dict__) + assert line["request_model"] == forged_model + assert "\n" not in message + assert message.startswith(f'token count - litellm_call_id="count log" request_model={json.dumps(forged_model)} ') diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index cb4d3f82611..cde025b10c8 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -329,6 +329,24 @@ def test_json_formatter_extra_component_not_overwritten(): assert obj["component"] == "auth-service", f"User-supplied component was overwritten, got {obj['component']!r}" +def test_json_formatter_extra_fields_win_over_a_dict_a_caller_put_in_the_message(): + record = logging.LogRecord( + name="LiteLLM Proxy", + level=logging.INFO, + pathname="", + lineno=0, + msg="token count - litellm_call_id=%s input_tokens=%s status_code=%s", + args=("{'input_tokens': 0, 'status_code': 200, 'injected': 'yes'}", None, 400), + exc_info=None, + ) + record.input_tokens = None + record.status_code = 400 + + obj = json.loads(JsonFormatter().format(record)) + + assert (obj["input_tokens"], obj["status_code"], obj["injected"]) == (None, 400, "yes") + + def test_initialize_loggers_with_handler_sets_propagate_false(): """ Test that the initialize_loggers_with_handler function sets propagate to False for all loggers diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0fc0d86b479..42388c54db7 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -19058,6 +19058,7 @@ export interface paths { * @description Args: * request: TokenCountRequest * call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs. + * litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs * * Returns: * TokenCountResponse @@ -76716,7 +76717,9 @@ export interface operations { query?: { call_endpoint?: boolean; }; - header?: never; + header?: { + "x-litellm-call-id"?: string | null; + }; path?: never; cookie?: never; };