From 34e037e79aca92efef66594003f9649967c29323 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:45:15 -0700 Subject: [PATCH 1/5] feat(proxy): log one line per token count call Each count call on /v1/messages/count_tokens, /utils/token_counter, /v1/responses/input_tokens and the Gemini countTokens routes now logs one INFO line with the litellm call id, the requested model, the model the count ran against, the count, the count source and the status code, so it joins the inference call that sends the same x-litellm-call-id --- .../proxy/anthropic_endpoints/endpoints.py | 1 + litellm/proxy/common_utils/token_count_log.py | 55 ++++++++++++ litellm/proxy/google_endpoints/endpoints.py | 8 +- litellm/proxy/proxy_server.py | 35 +++++++- .../proxy/response_api_endpoints/endpoints.py | 12 ++- .../anthropic_endpoints/test_endpoints.py | 83 +++++++++++++++++ tests/unit/proxy/conftest.py | 26 ++++++ .../test_google_api_endpoints.py | 44 ++++++++++ .../response_api_endpoints/test_endpoints.py | 54 +++++++++++- tests/unit/proxy/test_proxy_token_counter.py | 88 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +- 11 files changed, 402 insertions(+), 9 deletions(-) create mode 100644 litellm/proxy/common_utils/token_count_log.py diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index b12676e08f3..3e01bb01576 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -411,6 +411,7 @@ async def count_tokens( request=token_request, call_endpoint=True, messages_format="anthropic", + litellm_call_id=litellm_call_id, **_count_tokens_body_options(data, auto_drop_params), ) except HTTPException as e: diff --git a/litellm/proxy/common_utils/token_count_log.py b/litellm/proxy/common_utils/token_count_log.py new file mode 100644 index 00000000000..e46bf08fb99 --- /dev/null +++ b/litellm/proxy/common_utils/token_count_log.py @@ -0,0 +1,55 @@ +from collections.abc import Awaitable +from types import MappingProxyType +from typing import Final + +from fastapi import status + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.openai_error_payload import error_status_code +from litellm.types.token_count import TokenCountSource +from litellm.types.utils import TokenCountResponse + + +def _log_token_count( + litellm_call_id: str, + request_model: str, + counted: TokenCountResponse | None, + status_code: int, +) -> None: + model_used: Final = None if counted is None else counted.model_used + input_tokens: Final = None if counted is None else counted.total_tokens + count_source: Final[TokenCountSource | None] = None if counted is None else counted.count_source + verbose_proxy_logger.info( + "token count - litellm_call_id=%s request_model=%s model_used=%s input_tokens=%s count_source=%s " + "status_code=%s", + litellm_call_id, + request_model, + model_used, + input_tokens, + count_source, + status_code, + extra=MappingProxyType( + { + "litellm_call_id": litellm_call_id, + "request_model": request_model, + "model_used": model_used, + "input_tokens": input_tokens, + "count_source": count_source, + "status_code": status_code, + } + ), + ) + + +async def logged_token_count( + count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str +) -> TokenCountResponse: + try: + counted: Final = await count + except Exception as e: + _log_token_count( + litellm_call_id, request_model, None, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) + ) + raise + _log_token_count(litellm_call_id, request_model, counted, status.HTTP_200_OK) + return counted diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 3d1a9ab4cfe..6a3f1d5a197 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -21,6 +21,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export read_request_body, ) +from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER from litellm.proxy.common_utils.token_count_settings import count_source_fields from litellm.types.llms.vertex_ai import TokenCountDetailsResponse @@ -286,6 +287,7 @@ async def google_count_tokens(request: Request, model_name: str): from litellm.proxy.common_utils.http_parsing_utils import read_request_body from litellm.proxy.proxy_server import token_counter as internal_token_counter + litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)) data: Final = await read_request_body(request=request) try: token_request: Final = _google_count_tokens_request(model_name, data) @@ -298,13 +300,15 @@ async def google_count_tokens(request: Request, model_name: str): ) try: - token_response: Final = await internal_token_counter(request=token_request, call_endpoint=True) + token_response: Final = await internal_token_counter( + request=token_request, call_endpoint=True, litellm_call_id=litellm_call_id + ) except ProxyException as error: return _google_error_response(request, error) except HTTPException as error: return _google_error_response(request, proxy_exception_from_http_exception(error, {})) except Exception as error: - log_llm_api_exception(error, resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))) + log_llm_api_exception(error, litellm_call_id) return _google_error_response( request, ProxyException(message="Internal server error", type="internal_server_error", param=None, code=500), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c6d3e12a378..7b50467309e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -548,6 +548,7 @@ from litellm.proxy.common_utils.timezone_utils import ( get_budget_reset_settings, get_budget_reset_time, ) +from litellm.proxy.common_utils.token_count_log import logged_token_count from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, end_user_cache_key, @@ -15179,16 +15180,21 @@ def _system_message(system: object) -> ChatCompletionSystemMessage | None: dependencies=[Depends(user_api_key_auth)], response_model=TokenCountResponse, ) -async def token_counter(request: TokenCountRequest, call_endpoint: bool = False): +async def token_counter( + request: TokenCountRequest, + call_endpoint: bool = False, + litellm_call_id: Annotated[str | None, Header(alias=LITELLM_CALL_ID_HEADER)] = None, +): """ Args: request: TokenCountRequest call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs. + litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs Returns: TokenCountResponse """ - return await count_request_tokens(request=request, call_endpoint=call_endpoint) + return await count_request_tokens(request=request, call_endpoint=call_endpoint, litellm_call_id=litellm_call_id) async def count_request_tokens( @@ -15199,6 +15205,31 @@ async def count_request_tokens( context_management: JsonValue = None, request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}), responses_request: Mapping[str, object] | None = None, + litellm_call_id: str | None = None, +) -> TokenCountResponse: + return await logged_token_count( + _count_request_tokens( + request=request, + call_endpoint=call_endpoint, + messages_format=messages_format, + output_format=output_format, + context_management=context_management, + request_litellm_params=request_litellm_params, + responses_request=responses_request, + ), + litellm_call_id=resolve_litellm_call_id(litellm_call_id), + request_model=request.model, + ) + + +async def _count_request_tokens( + request: TokenCountRequest, + call_endpoint: bool, + messages_format: TokenCountMessagesFormat, + output_format: JsonValue, + context_management: JsonValue, + request_litellm_params: Mapping[str, JsonValue], + responses_request: Mapping[str, object] | None, ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index c918ea3f1d1..d3c4d4bc17f 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1391,7 +1391,12 @@ class _TokenCountPayload(TypedDict): class _TokenCounter(Protocol): def __call__( - self, *, request: TokenCountRequest, call_endpoint: bool, responses_request: Mapping[str, object] | None + self, + *, + request: TokenCountRequest, + call_endpoint: bool, + responses_request: Mapping[str, object] | None, + litellm_call_id: str | None, ) -> Awaitable[TokenCountResponse]: ... @@ -1683,7 +1688,10 @@ async def responses_input_tokens( ) try: token_response: Final = await token_counter( - request=token_request, call_endpoint=True, responses_request=responses_request + request=token_request, + call_endpoint=True, + responses_request=responses_request, + litellm_call_id=request.headers.get(LITELLM_CALL_ID_HEADER), ) except ProxyException as e: if e.type != _PROVIDER_COUNT_ERROR_TYPE: diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index c7bb05e5d30..3b78bbf7aa8 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -5,9 +5,11 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec import json import logging import unittest +from collections.abc import Callable, Mapping from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi.testclient import TestClient @@ -615,6 +617,87 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e assert counter.await_args.kwargs["request"].tools is None +def _count_on_claude( + monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str] +) -> httpx.Response: + from fastapi import FastAPI + + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.anthropic_endpoints.endpoints import router + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"} + monkeypatch.setattr( + proxy_server, "llm_router", Router(model_list=[{"model_name": "count-log-claude", "litellm_params": claude}]) + ) + monkeypatch.setattr( + anthropic_count_tokens_handler, + "get_async_httpx_client", + lambda *args, **kwargs: AsyncHTTPHandler(transport=httpx.MockTransport(peer)), + ) + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth() + return TestClient(app).post( + "/v1/messages/count_tokens", + json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]}, + headers=dict(headers), + ) + + +def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response(200, json={"input_tokens": 77}, request=request), + {"x-litellm-call-id": "count-log-msg-claude"}, + ) + + assert response.json() == {"input_tokens": 77, "litellm_count_source": "provider"}, response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-msg-claude", + "request_model": "count-log-claude", + "model_used": "claude-zeta-9", + "input_tokens": 77, + "count_source": "provider", + "status_code": 200, + }, + ) + + +def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + rejection: Final = { + "type": "error", + "error": {"type": "invalid_request_error", "message": "messages.0.content.0: invalid base64 data"}, + } + + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response(400, json=rejection, request=request), + {"x-litellm-call-id": "count-log-msg-error"}, + ) + + assert response.status_code == 400, response.text + assert response.json()["error"] == rejection["error"] + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-msg-error", + "request_model": "count-log-claude", + "model_used": None, + "input_tokens": None, + "count_source": None, + "status_code": 400, + }, + ) + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 58c9a5e4d76..79fb150c1cd 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -3,6 +3,7 @@ import asyncio import copy import inspect +import logging import os import tempfile import warnings @@ -17,6 +18,7 @@ from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server +from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceTypes from litellm.integrations.otel.model.payloads import ServiceSpanData from litellm.integrations.otel.model.spans import service_span_name @@ -424,6 +426,30 @@ def fresh_agent_read_through(monkeypatch): return read_through +TOKEN_COUNT_LOG_FIELDS: Final = ( + "litellm_call_id", + "request_model", + "model_used", + "input_tokens", + "count_source", + "status_code", +) + + +@pytest.fixture +def token_count_log_lines( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> Iterator[Callable[[], tuple[dict[str, object], ...]]]: + """The structured fields of every INFO token count line the proxy logger writes while the test runs.""" + monkeypatch.setattr(verbose_proxy_logger, "propagate", True) + with caplog.at_level(logging.INFO, logger=verbose_proxy_logger.name): + yield lambda: tuple( + {field: record.__dict__[field] for field in TOKEN_COUNT_LOG_FIELDS} + for record in caplog.records + if record.levelno == logging.INFO and "count_source" in record.__dict__ + ) + + @pytest.fixture def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]: """The ``postgres.{verb} {table}`` names OTel would render for every DB service event diff --git a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py index 2777d18a399..3756ecb847e 100644 --- a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py +++ b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py @@ -3,9 +3,11 @@ Test to verify the Google GenAI proxy API endpoints """ +from collections.abc import Callable from typing import Final from unittest.mock import AsyncMock, patch +import httpx import pytest from litellm.types.utils import TokenCountResponse @@ -240,3 +242,45 @@ def test_google_count_tokens_uses_normalized_total_when_provider_shape_differs() assert body["promptTokensDetails"] == [] assert body["litellm_count_source"] == "token_count_model" assert body["litellm_count_model"] == "us.anthropic.claude-opus-5-5" + + +@pytest.mark.parametrize("path", ["/v1beta/models/count-log-claude:countTokens", "/models/count-log-claude:countTokens"]) +def test_google_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], path: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm import Router + from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"} + monkeypatch.setattr( + proxy_server, "llm_router", Router(model_list=[{"model_name": "count-log-claude", "litellm_params": claude}]) + ) + monkeypatch.setattr( + anthropic_count_tokens_handler, + "get_async_httpx_client", + lambda *args, **kwargs: AsyncHTTPHandler( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"input_tokens": 77}, request=request) + ) + ), + ) + + response: Final = _build_test_client().post( + path, + json={"contents": [{"role": "user", "parts": [{"text": "Hello"}]}]}, + headers={"x-litellm-call-id": "count-log-gemini"}, + ) + + assert (response.status_code, response.json()["totalTokens"]) == (200, 77), response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-gemini", + "request_model": "count-log-claude", + "model_used": "claude-zeta-9", + "input_tokens": 77, + "count_source": "provider", + "status_code": 200, + }, + ) diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 32b77c3b564..8f9a9b9dd7e 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -7,7 +7,7 @@ import contextlib import dataclasses import json import unittest -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -19,6 +19,7 @@ from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.constants import EMPTY_MAPPING from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse @@ -4173,6 +4174,7 @@ class TestResponsesInputTokens: llm_router: litellm.Router | None = None, response_id_security: object | None = None, managed_files: object | None = None, + headers: Mapping[str, str] = EMPTY_MAPPING, ) -> tuple[Response, AsyncMock]: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -4205,7 +4207,7 @@ class TestResponsesInputTokens: app.dependency_overrides[_proxy_managed_files_hook] = lambda: managed_files try: client = TestClient(app) - response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876"}) + response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876", **headers}) return response, token_counter_mock finally: app.dependency_overrides.pop(user_api_key_auth, None) @@ -4450,6 +4452,54 @@ class TestResponsesInputTokens: assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text assert sent == [{"model": "gpt-4o-mini", "input": "hi"}] + @pytest.mark.parametrize( + "path", ["/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"] + ) + def test_count_logs_one_line_with_the_callers_call_id_model_and_count( + self, + monkeypatch: pytest.MonkeyPatch, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], + path: str, + ) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http + from litellm.proxy.proxy_server import count_request_tokens + + gpt: Final = {"model": "openai/gpt-4o-mini", "api_key": "test-key", "api_base": "https://count.test/v1"} + router: Final = litellm.Router(model_list=[{"model_name": "count-log-gpt", "litellm_params": gpt}]) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + monkeypatch.setattr( + openai_count_tokens_http, + "get_async_httpx_client", + lambda *args, **kwargs: AsyncHTTPHandler( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, json={"object": "response.input_tokens", "input_tokens": 7}, request=request + ) + ) + ), + ) + + response, _ = self._post_input_tokens( + {"model": "count-log-gpt", "input": "hi"}, + path=path, + counter=AsyncMock(side_effect=count_request_tokens), + llm_router=router, + headers={"x-litellm-call-id": "count-log-responses"}, + ) + + assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-responses", + "request_model": "count-log-gpt", + "model_used": "gpt-4o-mini", + "input_tokens": 7, + "count_source": "provider", + "status_code": 200, + }, + ) + def test_a_managed_file_the_key_cannot_read_answers_403_without_a_count(self) -> None: from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..955d0cc5759 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -10,6 +10,7 @@ import json import logging import threading import traceback +import uuid from collections.abc import Callable, Coroutine, Mapping from functools import partial from pathlib import Path @@ -364,6 +365,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -444,6 +446,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -1262,6 +1265,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1319,6 +1323,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1376,6 +1381,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): output_format=None, context_management=None, request_litellm_params=None, + litellm_call_id=None, ): raise ProxyException( message="Rate limit exceeded", @@ -3892,3 +3898,85 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r ] assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3 spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form")) + + +_COUNTED_HELLO: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"model": "count-log-gpt", "messages": [{"role": "user", "content": "Hello, how many tokens is this?"}]} +) + + +def _utils_token_counter( + monkeypatch: pytest.MonkeyPatch, body: Mapping[str, JsonValue], headers: Mapping[str, str] +) -> httpx.Response: + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + {"model_name": "count-log-gpt", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}} + ] + ), + ) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-test")) + return TestClient(app).post("/utils/token_counter", json=dict(body), headers=dict(headers)) + + +def test_utils_token_counter_logs_one_line_with_the_callers_call_id_model_and_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter(monkeypatch, _COUNTED_HELLO, {"x-litellm-call-id": "count-log-utils"}) + + assert response.status_code == 200, response.text + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-utils", + "request_model": "count-log-gpt", + "model_used": "gpt-4o-mini", + "input_tokens": response.json()["total_tokens"], + "count_source": "local_estimate", + "status_code": 200, + }, + ) + + +def test_utils_token_counter_logs_a_generated_call_id_when_the_client_sends_none( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter(monkeypatch, _COUNTED_HELLO, {}) + + assert response.status_code == 200, response.text + (line,) = token_count_log_lines() + assert str(uuid.UUID(str(line["litellm_call_id"]))) == line["litellm_call_id"] + + +def test_utils_token_counter_failed_count_logs_its_status_in_place_of_a_count( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter( + monkeypatch, {**_COUNTED_HELLO, "prompt": "and a prompt"}, {"x-litellm-call-id": "count-log-utils-error"} + ) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "prompt cannot be sent with messages or contents. Set drop_params to count them without the prompt", + "type": "invalid_request_error", + "param": "prompt", + "code": "400", + } + } + assert token_count_log_lines() == ( + { + "litellm_call_id": "count-log-utils-error", + "request_model": "count-log-gpt", + "model_used": None, + "input_tokens": None, + "count_source": None, + "status_code": 400, + }, + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e432c947116..6a828df89fd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -19017,6 +19017,7 @@ export interface paths { * @description Args: * request: TokenCountRequest * call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs. + * litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs * * Returns: * TokenCountResponse @@ -76240,7 +76241,9 @@ export interface operations { query?: { call_endpoint?: boolean; }; - header?: never; + header?: { + "x-litellm-call-id"?: string | null; + }; path?: never; cookie?: never; }; From 2b06373fd702609e8bca46ebe701e3807d450816 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:06:25 -0700 Subject: [PATCH 2/5] test(google_endpoints): give the count test's fake request a headers list --- tests/unit/proxy/google_endpoints/test_endpoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/proxy/google_endpoints/test_endpoints.py b/tests/unit/proxy/google_endpoints/test_endpoints.py index 17af52b5579..80e45edf897 100644 --- a/tests/unit/proxy/google_endpoints/test_endpoints.py +++ b/tests/unit/proxy/google_endpoints/test_endpoints.py @@ -94,6 +94,7 @@ async def test_proxy_gemini_to_openai_like_model_token_counting(): request=Request( scope={ "type": "http", + "headers": [], "parsed_body": ( ["contents"], {"contents": [{"parts": [{"text": "Hello, how are you?"}]}]}, From 8f362c08b55ab733a3cd8ae2ee0373a5b4501cd4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:19:36 -0700 Subject: [PATCH 3/5] fix(proxy): keep token count log lines one per call and trusted in JSON logs, and log Bedrock native counts Quote log values that carry whitespace, quotes, equals, or control characters so a model name can't split or forge a line. Let JsonFormatter record extras win over a dict parsed from the message. Log the Bedrock native count-tokens route through the same line. --- litellm/_logging.py | 10 +- litellm/proxy/common_utils/token_count_log.py | 92 ++++++++++++++----- .../llm_passthrough_endpoints.py | 19 ++-- .../test_llm_pass_through_endpoints.py | 84 ++++++++++++++++- tests/unit/proxy/test_proxy_token_counter.py | 16 ++++ tests/unit/test_logging.py | 18 ++++ 6 files changed, 204 insertions(+), 35 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index d08254db3fd..6bfca44aaa8 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -9,6 +9,7 @@ import sys from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter +from types import MappingProxyType from typing import Final, TextIO from urllib.parse import unquote @@ -755,17 +756,20 @@ class JsonFormatter(Formatter): # fields, and a message that merely happens to parse as JSON/dict (e.g. a proxy # log line dumping raw request headers) must never be able to claim them, even on # a record the filter hasn't stamped yet (no correlation context active for it). + extras: Final = MappingProxyType( + {key: value for key, value in _record_attributes(record).items() if key not in _NON_EXTRA_RECORD_ATTRS} + ) parsed = _try_parse_json_message(message_str) if parsed is None: parsed = _try_parse_embedded_python_dict(message_str) if parsed is not None: for key, value in parsed.items(): - if key not in json_record and key not in _RESERVED_CORRELATION_FIELDS: + if key not in json_record and key not in _RESERVED_CORRELATION_FIELDS and key not in extras: json_record[key] = value # Include extra attributes passed via logger.debug("msg", extra={...}) - for key, value in _record_attributes(record).items(): - if key not in _NON_EXTRA_RECORD_ATTRS and key not in json_record: + for key, value in extras.items(): + if key not in json_record: json_record[key] = value # trace_id/session_id are reserved: CorrelationContextFilter is the only diff --git a/litellm/proxy/common_utils/token_count_log.py b/litellm/proxy/common_utils/token_count_log.py index e46bf08fb99..82e5875a9a0 100644 --- a/litellm/proxy/common_utils/token_count_log.py +++ b/litellm/proxy/common_utils/token_count_log.py @@ -1,6 +1,8 @@ -from collections.abc import Awaitable +import json +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar from fastapi import status @@ -10,46 +12,88 @@ from litellm.types.token_count import TokenCountSource from litellm.types.utils import TokenCountResponse -def _log_token_count( - litellm_call_id: str, - request_model: str, - counted: TokenCountResponse | None, - status_code: int, -) -> None: - model_used: Final = None if counted is None else counted.model_used - input_tokens: Final = None if counted is None else counted.total_tokens - count_source: Final[TokenCountSource | None] = None if counted is None else counted.count_source +def _logfmt_value(value: str | None) -> str: + if value is None: + return "None" + bare: Final = value != "" and value.isprintable() and not any(ch.isspace() or ch in '"=' for ch in value) + return value if bare else json.dumps(value) + + +_Counted: Final = TypeVar("_Counted") +_ProviderCount: Final = TypeVar("_ProviderCount", bound=Mapping[str, object]) + + +@dataclass(frozen=True, slots=True) +class _CountFields: + model_used: str | None + input_tokens: int | None + count_source: TokenCountSource | None + + +_FAILED_COUNT: Final = _CountFields(model_used=None, input_tokens=None, count_source=None) + + +def _log_token_count(litellm_call_id: str, request_model: str, fields: _CountFields, status_code: int) -> None: verbose_proxy_logger.info( "token count - litellm_call_id=%s request_model=%s model_used=%s input_tokens=%s count_source=%s " "status_code=%s", - litellm_call_id, - request_model, - model_used, - input_tokens, - count_source, + _logfmt_value(litellm_call_id), + _logfmt_value(request_model), + _logfmt_value(fields.model_used), + fields.input_tokens, + fields.count_source, status_code, extra=MappingProxyType( { "litellm_call_id": litellm_call_id, "request_model": request_model, - "model_used": model_used, - "input_tokens": input_tokens, - "count_source": count_source, + "model_used": fields.model_used, + "input_tokens": fields.input_tokens, + "count_source": fields.count_source, "status_code": status_code, } ), ) -async def logged_token_count( - count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str -) -> TokenCountResponse: +async def _logged( + count: Awaitable[_Counted], + litellm_call_id: str, + request_model: str, + fields_of: Callable[[_Counted], _CountFields], +) -> _Counted: try: counted: Final = await count except Exception as e: _log_token_count( - litellm_call_id, request_model, None, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) + litellm_call_id, request_model, _FAILED_COUNT, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) ) raise - _log_token_count(litellm_call_id, request_model, counted, status.HTTP_200_OK) + _log_token_count(litellm_call_id, request_model, fields_of(counted), status.HTTP_200_OK) return counted + + +def _token_count_response_fields(counted: TokenCountResponse) -> _CountFields: + return _CountFields( + model_used=counted.model_used, input_tokens=counted.total_tokens, count_source=counted.count_source + ) + + +async def logged_token_count( + count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str +) -> TokenCountResponse: + return await _logged(count, litellm_call_id, request_model, _token_count_response_fields) + + +async def logged_provider_token_count( + count: Awaitable[_ProviderCount], litellm_call_id: str, request_model: str, model_used: str +) -> _ProviderCount: + def provider_fields(counted: _ProviderCount) -> _CountFields: + input_tokens: Final = counted.get("input_tokens") + return _CountFields( + model_used=model_used, + input_tokens=input_tokens if isinstance(input_tokens, int) and not isinstance(input_tokens, bool) else None, + count_source="provider", + ) + + return await _logged(count, litellm_call_id, request_model, provider_fields) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 16fbd1de376..65dd4d7630e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -87,6 +87,7 @@ from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, model_deployments_support_stream_options, open_sse_before_first_byte, + resolve_litellm_call_id, ) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -99,10 +100,12 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg safe_get_request_headers, safe_set_request_parsed_body, ) +from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) +from litellm.proxy.common_utils.token_count_log import logged_provider_token_count from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( stream_usage_relay, @@ -1291,15 +1294,17 @@ async def handle_bedrock_count_tokens( verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params) verbose_proxy_logger.debug("Resolved model: %s", resolved_model) - # Handle the count tokens request - result: Final = await handler.handle_count_tokens_request( - request_data=request_body, - litellm_params=litellm_params, - resolved_model=resolved_model, + return await logged_provider_token_count( + handler.handle_count_tokens_request( + request_data=request_body, + litellm_params=litellm_params, + resolved_model=resolved_model, + ), + litellm_call_id=resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)), + request_model=model, + model_used=resolved_model.partition("/")[2] or resolved_model, ) - return result - except BedrockError as e: # Convert BedrockError to HTTPException for FastAPI verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 0186a33dbc4..c44e5d5d0e3 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2142,7 +2142,6 @@ class TestBedrockPassthroughAwsErrors: assert exc_info.value.aws_error_type == "ValidationException" assert (exc_info.value.error.message, exc_info.value.error.code) == (message, status_code) - @pytest.mark.asyncio async def test_bedrock_request_check_is_raised_with_the_aws_error_type_it_carries( self, monkeypatch: pytest.MonkeyPatch @@ -2183,6 +2182,7 @@ class TestBedrockPassthroughAwsErrors: await self._answer(error) assert (exc_info.value.code, exc_info.value.headers.get("x-model-group")) == ("403", "modèle") + @pytest.mark.asyncio @pytest.mark.parametrize( ("error", "message", "code"), @@ -7249,6 +7249,88 @@ async def test_bedrock_count_tokens_error_forwards_provider_headers(): assert exc_info.value.headers["llm_provider-x-amzn-requestid"] == "req-count-tokens-500" +_BEDROCK_COUNT_ALIAS: Final = "count-log-bedrock" +_BEDROCK_COUNT_MODEL: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +async def _bedrock_native_count(monkeypatch: pytest.MonkeyPatch, counted: AsyncMock) -> dict[str, object]: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + handle_bedrock_count_tokens, + ) + + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + litellm.Router( + model_list=[{"model_name": _BEDROCK_COUNT_ALIAS, "litellm_params": {"model": _BEDROCK_COUNT_MODEL}}] + ), + ) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": f"/bedrock/model/{_BEDROCK_COUNT_ALIAS}/count-tokens", + "headers": [(b"x-litellm-call-id", b"bedrock-native-count")], + } + ) + with patch( # test-quality-ok: the native route counts through this handler, which signs and calls AWS + "litellm.llms.bedrock.count_tokens.handler.BedrockCountTokensHandler.handle_count_tokens_request", + new=counted, + ): + return await handle_bedrock_count_tokens( + endpoint=f"model/{_BEDROCK_COUNT_ALIAS}/count-tokens", + request=request, + fastapi_response=Response(), + user_api_key_dict=MagicMock(), + request_body={"model": _BEDROCK_COUNT_ALIAS, "messages": [{"role": "user", "content": "hi"}]}, + ) + + +@pytest.mark.asyncio +async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_model( + monkeypatch: pytest.MonkeyPatch, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + result: Final = await _bedrock_native_count(monkeypatch, AsyncMock(return_value={"input_tokens": 12})) + + assert result == {"input_tokens": 12} + assert token_count_log_lines() == ( + { + "litellm_call_id": "bedrock-native-count", + "request_model": _BEDROCK_COUNT_ALIAS, + "model_used": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "input_tokens": 12, + "count_source": "provider", + "status_code": 200, + }, + ) + + +@pytest.mark.asyncio +async def test_bedrock_native_count_tokens_logs_the_provider_status_when_the_count_fails( + monkeypatch: pytest.MonkeyPatch, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + from litellm.llms.bedrock.common_utils import BedrockError + + rejected: Final = AsyncMock( + side_effect=BedrockError(status_code=400, message="The provided model doesn't support counting tokens.") + ) + with pytest.raises(HTTPException) as exc_info: + await _bedrock_native_count(monkeypatch, rejected) + + assert exc_info.value.status_code == 400 + assert token_count_log_lines() == ( + { + "litellm_call_id": "bedrock-native-count", + "request_model": _BEDROCK_COUNT_ALIAS, + "model_used": None, + "input_tokens": None, + "count_source": None, + "status_code": 400, + }, + ) + + class _AzureGroupRouter: def __init__(self, captured: list[dict]) -> None: self.captured = captured diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 955d0cc5759..49da84e015b 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -3980,3 +3980,19 @@ def test_utils_token_counter_failed_count_logs_its_status_in_place_of_a_count( "status_code": 400, }, ) + + +def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newline( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + forged_model: Final = "count-log-gpt\ntoken count - litellm_call_id=forged input_tokens=1" + + _utils_token_counter(monkeypatch, {**_COUNTED_HELLO, "model": forged_model}, {"x-litellm-call-id": "count log"}) + + (line,) = token_count_log_lines() + (message,) = (record.getMessage() for record in caplog.records if "count_source" in record.__dict__) + assert line["request_model"] == forged_model + assert "\n" not in message + assert message.startswith(f'token count - litellm_call_id="count log" request_model={json.dumps(forged_model)} ') diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index cb4d3f82611..cde025b10c8 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -329,6 +329,24 @@ def test_json_formatter_extra_component_not_overwritten(): assert obj["component"] == "auth-service", f"User-supplied component was overwritten, got {obj['component']!r}" +def test_json_formatter_extra_fields_win_over_a_dict_a_caller_put_in_the_message(): + record = logging.LogRecord( + name="LiteLLM Proxy", + level=logging.INFO, + pathname="", + lineno=0, + msg="token count - litellm_call_id=%s input_tokens=%s status_code=%s", + args=("{'input_tokens': 0, 'status_code': 200, 'injected': 'yes'}", None, 400), + exc_info=None, + ) + record.input_tokens = None + record.status_code = 400 + + obj = json.loads(JsonFormatter().format(record)) + + assert (obj["input_tokens"], obj["status_code"], obj["injected"]) == (None, 400, "yes") + + def test_initialize_loggers_with_handler_sets_propagate_false(): """ Test that the initialize_loggers_with_handler function sets propagate to False for all loggers From a59f5955984558c9a8f90b964986ebeee4ed03d5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:45:41 -0700 Subject: [PATCH 4/5] fix(proxy): escape braces in token count log values, log the Bedrock model AWS counts, and skip the line when there is nothing to count --- .../bedrock/count_tokens/transformation.py | 11 ++-- litellm/proxy/_types.py | 4 +- litellm/proxy/common_utils/token_count_log.py | 9 ++- .../llm_passthrough_endpoints.py | 2 +- litellm/proxy/proxy_server.py | 8 +-- .../test_llm_pass_through_endpoints.py | 58 ++++++++++++------- tests/unit/proxy/test_proxy_token_counter.py | 34 ++++++++++- 7 files changed, 87 insertions(+), 39 deletions(-) diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index 7cca4f69acc..6fa5ffbaa1f 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -205,6 +205,10 @@ class BedrockCountTokensConfig(BaseAWSLLM): encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode() return {"input": {"invokeModel": {"body": encoded_body}}} + @staticmethod + def count_tokens_model_id(model: str) -> str: + return get_bedrock_base_model(model).removeprefix("bedrock/") + def get_bedrock_count_tokens_endpoint( self, model: str, @@ -224,12 +228,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): Returns: Complete endpoint URL for CountTokens API """ - # Use existing LiteLLM function to get the base model ID (removes region prefix) - model_id = get_bedrock_base_model(model) - - # Remove bedrock/ prefix if present - model_id = model_id.removeprefix("bedrock/") # Remove "bedrock/" prefix - encoded_model_id: Final = self.encode_model_id(model_id=model_id) + encoded_model_id: Final = self.encode_model_id(model_id=self.count_tokens_model_id(model)) base_url, _ = self.get_runtime_endpoint( api_base=api_base, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 315533d6d0f..a6de1caddd7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3448,12 +3448,12 @@ class LiteLLM_SpendLogs_ResponseObject(LiteLLMPydanticObjectBase): class TokenCountRequest(LiteLLMPydanticObjectBase): model: str prompt: str | None = None - messages: list[dict] | None = None + messages: list[dict[Any, Any]] | None = None """ Anthropic token counting endpoint uses /messages """ - contents: list[dict] | None = None + contents: list[dict[Any, Any]] | None = None """ Google /countTokens endpoint expects contents to be a list of dicts with the following structure: """ diff --git a/litellm/proxy/common_utils/token_count_log.py b/litellm/proxy/common_utils/token_count_log.py index 82e5875a9a0..e668764e2b0 100644 --- a/litellm/proxy/common_utils/token_count_log.py +++ b/litellm/proxy/common_utils/token_count_log.py @@ -1,4 +1,5 @@ import json +import re from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from types import MappingProxyType @@ -11,12 +12,16 @@ from litellm.proxy.common_utils.openai_error_payload import error_status_code from litellm.types.token_count import TokenCountSource from litellm.types.utils import TokenCountResponse +_BARE_LOGFMT_VALUE: Final = re.compile(r"[\w.:/@+-]+", re.ASCII) +_JSON_ESCAPED_BRACES: Final = str.maketrans({"{": "\\u007b", "}": "\\u007d"}) + def _logfmt_value(value: str | None) -> str: if value is None: return "None" - bare: Final = value != "" and value.isprintable() and not any(ch.isspace() or ch in '"=' for ch in value) - return value if bare else json.dumps(value) + if _BARE_LOGFMT_VALUE.fullmatch(value): + return value + return json.dumps(value).translate(_JSON_ESCAPED_BRACES) _Counted: Final = TypeVar("_Counted") diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 65dd4d7630e..5979d3d3418 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1302,7 +1302,7 @@ async def handle_bedrock_count_tokens( ), litellm_call_id=resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)), request_model=model, - model_used=resolved_model.partition("/")[2] or resolved_model, + model_used=handler.count_tokens_model_id(resolved_model), ) except BedrockError as e: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7b50467309e..c197e6b61e9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15207,6 +15207,8 @@ async def count_request_tokens( responses_request: Mapping[str, object] | None = None, litellm_call_id: str | None = None, ) -> TokenCountResponse: + if request.prompt is None and request.messages is None and request.contents is None: + raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided") return await logged_token_count( _count_request_tokens( request=request, @@ -15237,12 +15239,6 @@ async def _count_request_tokens( tools: Final = request.tools system: Final = request.system - ######################################################### - # Validate request - ######################################################### - if prompt is None and messages is None and contents is None: - raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided") - deployment: dict[str, Any] | None = None litellm_model_name = None model_info: ModelMapInfo | None = None diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index c44e5d5d0e3..960fa0fae85 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7253,17 +7253,33 @@ _BEDROCK_COUNT_ALIAS: Final = "count-log-bedrock" _BEDROCK_COUNT_MODEL: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" -async def _bedrock_native_count(monkeypatch: pytest.MonkeyPatch, counted: AsyncMock) -> dict[str, object]: +async def _bedrock_native_count( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, aws_reply: httpx.Response +) -> dict[str, object]: from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( handle_bedrock_count_tokens, ) + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", litellm.Router( - model_list=[{"model_name": _BEDROCK_COUNT_ALIAS, "litellm_params": {"model": _BEDROCK_COUNT_MODEL}}] + model_list=[ + { + "model_name": _BEDROCK_COUNT_ALIAS, + "litellm_params": { + "model": _BEDROCK_COUNT_MODEL, + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + }, + } + ] ), ) + respx_mock.post(host="bedrock-runtime.us-east-1.amazonaws.com", path__regex=r"/count-tokens$").mock( + return_value=aws_reply + ) request: Final = Request( { "type": "http", @@ -7272,32 +7288,34 @@ async def _bedrock_native_count(monkeypatch: pytest.MonkeyPatch, counted: AsyncM "headers": [(b"x-litellm-call-id", b"bedrock-native-count")], } ) - with patch( # test-quality-ok: the native route counts through this handler, which signs and calls AWS - "litellm.llms.bedrock.count_tokens.handler.BedrockCountTokensHandler.handle_count_tokens_request", - new=counted, - ): - return await handle_bedrock_count_tokens( - endpoint=f"model/{_BEDROCK_COUNT_ALIAS}/count-tokens", - request=request, - fastapi_response=Response(), - user_api_key_dict=MagicMock(), - request_body={"model": _BEDROCK_COUNT_ALIAS, "messages": [{"role": "user", "content": "hi"}]}, - ) + return await handle_bedrock_count_tokens( + endpoint=f"model/{_BEDROCK_COUNT_ALIAS}/count-tokens", + request=request, + fastapi_response=Response(), + user_api_key_dict=MagicMock(), + request_body={"model": _BEDROCK_COUNT_ALIAS, "messages": [{"role": "user", "content": "hi"}]}, + ) @pytest.mark.asyncio async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_model( monkeypatch: pytest.MonkeyPatch, + httpx_transport: None, + respx_mock: respx.MockRouter, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], ) -> None: - result: Final = await _bedrock_native_count(monkeypatch, AsyncMock(return_value={"input_tokens": 12})) + result: Final = await _bedrock_native_count( + monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12}) + ) assert result == {"input_tokens": 12} + assert len(respx_mock.calls) == 1 + assert respx_mock.calls.last.request.url.path == "/model/anthropic.claude-haiku-4-5-20251001-v1:0/count-tokens" assert token_count_log_lines() == ( { "litellm_call_id": "bedrock-native-count", "request_model": _BEDROCK_COUNT_ALIAS, - "model_used": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "model_used": "anthropic.claude-haiku-4-5-20251001-v1:0", "input_tokens": 12, "count_source": "provider", "status_code": 200, @@ -7308,15 +7326,13 @@ async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_mod @pytest.mark.asyncio async def test_bedrock_native_count_tokens_logs_the_provider_status_when_the_count_fails( monkeypatch: pytest.MonkeyPatch, + httpx_transport: None, + respx_mock: respx.MockRouter, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], ) -> None: - from litellm.llms.bedrock.common_utils import BedrockError - - rejected: Final = AsyncMock( - side_effect=BedrockError(status_code=400, message="The provided model doesn't support counting tokens.") - ) + rejected: Final = httpx.Response(400, json={"message": "The provided model doesn't support counting tokens."}) with pytest.raises(HTTPException) as exc_info: - await _bedrock_native_count(monkeypatch, rejected) + await _bedrock_native_count(monkeypatch, respx_mock, rejected) assert exc_info.value.status_code == 400 assert token_count_log_lines() == ( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 49da84e015b..2086a937403 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -41,7 +41,7 @@ from starlette.types import Message import litellm from litellm import Router -from litellm._logging import verbose_proxy_logger +from litellm._logging import JsonFormatter, verbose_proxy_logger from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.common_utils import BedrockError @@ -3982,6 +3982,38 @@ def test_utils_token_counter_failed_count_logs_its_status_in_place_of_a_count( ) +def test_utils_token_counter_request_with_nothing_to_count_writes_no_line( + monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] +) -> None: + response: Final = _utils_token_counter( + monkeypatch, {"model": "count-log-gpt"}, {"x-litellm-call-id": "count-log-utils-empty"} + ) + + assert response.status_code == 400, response.text + assert token_count_log_lines() == () + + +def test_utils_token_counter_json_line_keeps_its_own_fields_when_the_call_id_is_a_dict( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + forged_call_id: Final = "{'component':'forged-comp','logger':'forged.py:1','injected_key':'injected'}" + + _utils_token_counter(monkeypatch, _COUNTED_HELLO, {"x-litellm-call-id": forged_call_id}) + + (line,) = token_count_log_lines() + (record,) = (record for record in caplog.records if "count_source" in record.__dict__) + rendered: Final = json.loads(JsonFormatter().format(record)) + assert line["litellm_call_id"] == forged_call_id + assert {key: rendered.get(key) for key in ("litellm_call_id", "component", "logger", "injected_key")} == { + "litellm_call_id": forged_call_id, + "component": verbose_proxy_logger.name, + "logger": f"{record.filename}:{record.lineno}", + "injected_key": None, + } + + def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newline( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, From 1c109685efae0d89c3986d3dae65b3b8acb09f5f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:49:42 -0700 Subject: [PATCH 5/5] fix(proxy): log the deployment id the count routed to on every token count line --- litellm/proxy/common_utils/token_count_log.py | 50 ++++++++++++------- .../llm_passthrough_endpoints.py | 27 +++++----- litellm/proxy/proxy_server.py | 35 ++++++++----- .../anthropic_endpoints/test_endpoints.py | 19 ++++++- tests/unit/proxy/conftest.py | 1 + .../test_google_api_endpoints.py | 13 ++++- .../test_llm_pass_through_endpoints.py | 3 ++ .../response_api_endpoints/test_endpoints.py | 11 +++- tests/unit/proxy/test_proxy_token_counter.py | 8 ++- 9 files changed, 117 insertions(+), 50 deletions(-) diff --git a/litellm/proxy/common_utils/token_count_log.py b/litellm/proxy/common_utils/token_count_log.py index e668764e2b0..3d92fa31ada 100644 --- a/litellm/proxy/common_utils/token_count_log.py +++ b/litellm/proxy/common_utils/token_count_log.py @@ -38,20 +38,37 @@ class _CountFields: _FAILED_COUNT: Final = _CountFields(model_used=None, input_tokens=None, count_source=None) -def _log_token_count(litellm_call_id: str, request_model: str, fields: _CountFields, status_code: int) -> None: +@dataclass(frozen=True, slots=True) +class _CountRequest: + litellm_call_id: str + request_model: str + model_id: str | None + + +def deployment_model_id(deployment: Mapping[str, object] | None) -> str | None: + match deployment: + case {"model_info": {"id": str() as model_id}}: + return model_id + case _: + return None + + +def _log_token_count(request: _CountRequest, fields: _CountFields, status_code: int) -> None: verbose_proxy_logger.info( - "token count - litellm_call_id=%s request_model=%s model_used=%s input_tokens=%s count_source=%s " - "status_code=%s", - _logfmt_value(litellm_call_id), - _logfmt_value(request_model), + "token count - litellm_call_id=%s request_model=%s model_id=%s model_used=%s input_tokens=%s " + "count_source=%s status_code=%s", + _logfmt_value(request.litellm_call_id), + _logfmt_value(request.request_model), + _logfmt_value(request.model_id), _logfmt_value(fields.model_used), fields.input_tokens, fields.count_source, status_code, extra=MappingProxyType( { - "litellm_call_id": litellm_call_id, - "request_model": request_model, + "litellm_call_id": request.litellm_call_id, + "request_model": request.request_model, + "model_id": request.model_id, "model_used": fields.model_used, "input_tokens": fields.input_tokens, "count_source": fields.count_source, @@ -62,19 +79,14 @@ def _log_token_count(litellm_call_id: str, request_model: str, fields: _CountFie async def _logged( - count: Awaitable[_Counted], - litellm_call_id: str, - request_model: str, - fields_of: Callable[[_Counted], _CountFields], + count: Awaitable[_Counted], request: _CountRequest, fields_of: Callable[[_Counted], _CountFields] ) -> _Counted: try: counted: Final = await count except Exception as e: - _log_token_count( - litellm_call_id, request_model, _FAILED_COUNT, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) - ) + _log_token_count(request, _FAILED_COUNT, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR)) raise - _log_token_count(litellm_call_id, request_model, fields_of(counted), status.HTTP_200_OK) + _log_token_count(request, fields_of(counted), status.HTTP_200_OK) return counted @@ -85,13 +97,13 @@ def _token_count_response_fields(counted: TokenCountResponse) -> _CountFields: async def logged_token_count( - count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str + count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str, model_id: str | None ) -> TokenCountResponse: - return await _logged(count, litellm_call_id, request_model, _token_count_response_fields) + return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), _token_count_response_fields) async def logged_provider_token_count( - count: Awaitable[_ProviderCount], litellm_call_id: str, request_model: str, model_used: str + count: Awaitable[_ProviderCount], litellm_call_id: str, request_model: str, model_id: str | None, model_used: str ) -> _ProviderCount: def provider_fields(counted: _ProviderCount) -> _CountFields: input_tokens: Final = counted.get("input_tokens") @@ -101,4 +113,4 @@ async def logged_provider_token_count( count_source="provider", ) - return await _logged(count, litellm_call_id, request_model, provider_fields) + return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), provider_fields) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 5979d3d3418..cba0f93c81b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -105,7 +105,7 @@ from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) -from litellm.proxy.common_utils.token_count_log import logged_provider_token_count +from litellm.proxy.common_utils.token_count_log import deployment_model_id, logged_provider_token_count from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( stream_usage_relay, @@ -1275,21 +1275,19 @@ async def handle_bedrock_count_tokens( litellm_params: Final = {"user_api_key_dict": user_api_key_dict} resolved_model = model # Default fallback - if llm_router: - deployments: Final = llm_router.get_model_list(model_name=model) - if deployments and len(deployments) > 0: - # Get the first matching deployment - deployment: Final = deployments[0] - model_litellm_params: Final = deployment.get("litellm_params", {}) + deployments: Final = llm_router.get_model_list(model_name=model) if llm_router else None + deployment: Final = deployments[0] if deployments else None + if deployment is not None: + model_litellm_params: Final = deployment.get("litellm_params", {}) - # Get the resolved model ID from the configuration - if "model" in model_litellm_params: - resolved_model = model_litellm_params["model"] + # Get the resolved model ID from the configuration + if "model" in model_litellm_params: + resolved_model = model_litellm_params["model"] - # Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery - for key, value in model_litellm_params.items(): - if key != "user_api_key_dict": # Don't overwrite user_api_key_dict - litellm_params[key] = value + # Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery + for key, value in model_litellm_params.items(): + if key != "user_api_key_dict": # Don't overwrite user_api_key_dict + litellm_params[key] = value verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params) verbose_proxy_logger.debug("Resolved model: %s", resolved_model) @@ -1302,6 +1300,7 @@ async def handle_bedrock_count_tokens( ), litellm_call_id=resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)), request_model=model, + model_id=deployment_model_id(deployment), model_used=handler.count_tokens_model_id(resolved_model), ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c197e6b61e9..55c72aef7d4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -548,7 +548,7 @@ from litellm.proxy.common_utils.timezone_utils import ( get_budget_reset_settings, get_budget_reset_time, ) -from litellm.proxy.common_utils.token_count_log import logged_token_count +from litellm.proxy.common_utils.token_count_log import deployment_model_id, logged_token_count from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, end_user_cache_key, @@ -15209,9 +15209,11 @@ async def count_request_tokens( ) -> TokenCountResponse: if request.prompt is None and request.messages is None and request.contents is None: raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided") + routed: Final = await _routed_count_deployment(request.model, responses_request) return await logged_token_count( _count_request_tokens( request=request, + routed=routed, call_endpoint=call_endpoint, messages_format=messages_format, output_format=output_format, @@ -15221,11 +15223,29 @@ async def count_request_tokens( ), litellm_call_id=resolve_litellm_call_id(litellm_call_id), request_model=request.model, + model_id=deployment_model_id(routed), ) +async def _routed_count_deployment( + model: str, responses_request: Mapping[str, object] | None +) -> Mapping[str, object] | None: + if llm_router is None: + return None + try: + return await llm_router.async_get_available_deployment( + model=model, request_kwargs=dict(_responses_routing_kwargs(responses_request)) + ) + except Exception: + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment" + ) + return None + + async def _count_request_tokens( request: TokenCountRequest, + routed: Mapping[str, object] | None, call_endpoint: bool, messages_format: TokenCountMessagesFormat, output_format: JsonValue, @@ -15239,20 +15259,9 @@ async def _count_request_tokens( tools: Final = request.tools system: Final = request.system - deployment: dict[str, Any] | None = None + deployment: dict[str, Any] | None = None if routed is None else dict(routed) litellm_model_name = None model_info: ModelMapInfo | None = None - if llm_router is not None: - # get 1 deployment corresponding to the model - try: - deployment = await llm_router.async_get_available_deployment( - model=request.model, - request_kwargs=dict(_responses_routing_kwargs(responses_request)), - ) - except Exception: - verbose_proxy_logger.exception( - "litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment" - ) if deployment is not None: litellm_model_name = deployment.get("litellm_params", {}).get("model") model_info = deployment.get("model_info", {}) diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 3b78bbf7aa8..c041ebe56a8 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -631,7 +631,22 @@ def _count_on_claude( claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"} monkeypatch.setattr( - proxy_server, "llm_router", Router(model_list=[{"model_name": "count-log-claude", "litellm_params": claude}]) + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "count-log-claude", + "litellm_params": {**claude, "order": 2}, + "model_info": {"id": "count-log-fallback-deployment"}, + }, + { + "model_name": "count-log-claude", + "litellm_params": {**claude, "order": 1}, + "model_info": {"id": "count-log-routed-deployment"}, + }, + ] + ), ) monkeypatch.setattr( anthropic_count_tokens_handler, @@ -662,6 +677,7 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( { "litellm_call_id": "count-log-msg-claude", "request_model": "count-log-claude", + "model_id": "count-log-routed-deployment", "model_used": "claude-zeta-9", "input_tokens": 77, "count_source": "provider", @@ -690,6 +706,7 @@ def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_er { "litellm_call_id": "count-log-msg-error", "request_model": "count-log-claude", + "model_id": "count-log-routed-deployment", "model_used": None, "input_tokens": None, "count_source": None, diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 79fb150c1cd..e13eddef43b 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -429,6 +429,7 @@ def fresh_agent_read_through(monkeypatch): TOKEN_COUNT_LOG_FIELDS: Final = ( "litellm_call_id", "request_model", + "model_id", "model_used", "input_tokens", "count_source", diff --git a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py index 3756ecb847e..bd6315ce481 100644 --- a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py +++ b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py @@ -255,7 +255,17 @@ def test_google_count_tokens_logs_one_line_with_the_callers_call_id_model_and_co claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"} monkeypatch.setattr( - proxy_server, "llm_router", Router(model_list=[{"model_name": "count-log-claude", "litellm_params": claude}]) + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "count-log-claude", + "litellm_params": claude, + "model_info": {"id": "count-log-gemini-deployment"}, + } + ] + ), ) monkeypatch.setattr( anthropic_count_tokens_handler, @@ -278,6 +288,7 @@ def test_google_count_tokens_logs_one_line_with_the_callers_call_id_model_and_co { "litellm_call_id": "count-log-gemini", "request_model": "count-log-claude", + "model_id": "count-log-gemini-deployment", "model_used": "claude-zeta-9", "input_tokens": 77, "count_source": "provider", diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 960fa0fae85..3f6830f10dd 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7273,6 +7273,7 @@ async def _bedrock_native_count( "aws_access_key_id": "AKIATESTACCESSKEY", "aws_secret_access_key": "test-secret", }, + "model_info": {"id": "count-log-bedrock-deployment"}, } ] ), @@ -7315,6 +7316,7 @@ async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_mod { "litellm_call_id": "bedrock-native-count", "request_model": _BEDROCK_COUNT_ALIAS, + "model_id": "count-log-bedrock-deployment", "model_used": "anthropic.claude-haiku-4-5-20251001-v1:0", "input_tokens": 12, "count_source": "provider", @@ -7339,6 +7341,7 @@ async def test_bedrock_native_count_tokens_logs_the_provider_status_when_the_cou { "litellm_call_id": "bedrock-native-count", "request_model": _BEDROCK_COUNT_ALIAS, + "model_id": "count-log-bedrock-deployment", "model_used": None, "input_tokens": None, "count_source": None, diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 8f9a9b9dd7e..c6d78838b2e 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -4466,7 +4466,15 @@ class TestResponsesInputTokens: from litellm.proxy.proxy_server import count_request_tokens gpt: Final = {"model": "openai/gpt-4o-mini", "api_key": "test-key", "api_base": "https://count.test/v1"} - router: Final = litellm.Router(model_list=[{"model_name": "count-log-gpt", "litellm_params": gpt}]) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "count-log-gpt", + "litellm_params": gpt, + "model_info": {"id": "count-log-responses-deployment"}, + } + ] + ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) monkeypatch.setattr( openai_count_tokens_http, @@ -4493,6 +4501,7 @@ class TestResponsesInputTokens: { "litellm_call_id": "count-log-responses", "request_model": "count-log-gpt", + "model_id": "count-log-responses-deployment", "model_used": "gpt-4o-mini", "input_tokens": 7, "count_source": "provider", diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 2086a937403..fa959d86510 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -3918,7 +3918,11 @@ def _utils_token_counter( "llm_router", Router( model_list=[ - {"model_name": "count-log-gpt", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}} + { + "model_name": "count-log-gpt", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + "model_info": {"id": "count-log-utils-deployment"}, + } ] ), ) @@ -3936,6 +3940,7 @@ def test_utils_token_counter_logs_one_line_with_the_callers_call_id_model_and_co { "litellm_call_id": "count-log-utils", "request_model": "count-log-gpt", + "model_id": "count-log-utils-deployment", "model_used": "gpt-4o-mini", "input_tokens": response.json()["total_tokens"], "count_source": "local_estimate", @@ -3974,6 +3979,7 @@ def test_utils_token_counter_failed_count_logs_its_status_in_place_of_a_count( { "litellm_call_id": "count-log-utils-error", "request_model": "count-log-gpt", + "model_id": "count-log-utils-deployment", "model_used": None, "input_tokens": None, "count_source": None,