From 717e985751db191721f0c7ea75df620eeb1d644d Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Wed, 17 Jun 2026 19:30:23 +0800 Subject: [PATCH] fix(anthropic): strip LiteLLM-injected total_tokens from /v1/messages response (#30382) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(anthropic): strip LiteLLM-injected total_tokens from /v1/messages response The non-streaming /v1/messages response carries a LiteLLM-injected usage.total_tokens = input_tokens + output_tokens that is not part of the Anthropic API spec. This caused three problems: 1. Shape divergence with streaming on the same endpoint. message_delta.usage in the SSE path never carries total_tokens. Clients parsing both paths get two different schemas from one endpoint. 2. Shape divergence with upstream. Direct calls to https://api.anthropic.com/v1/messages return no total_tokens field, so clients using the official Anthropic SDK couldn't rely on it, and clients that did rely on the LiteLLM-injected one broke when bypassing the proxy. 3. Numerical misuse. total = input + output undercounts when cache_read_input_tokens and cache_creation_input_tokens are non-zero, because cache tokens are reported in their own fields. A 100k-token cached prompt with 1 non-cache input token + 200 output tokens reports total_tokens = 201, off by ~99.8% from any reasonable definition of "total." Fix: add _strip_total_tokens_from_anthropic_response in litellm/proxy/anthropic_endpoints/endpoints.py and invoke it in the success path of anthropic_response right before returning. Only mutates dict-shaped responses; streaming (which already lacks the field) is left untouched. spend_logs / Prometheus continue to compute total_tokens internally for billing — this fix only strips the field from the wire response. Scope: only the Anthropic passthrough endpoint /v1/messages. The OpenAI-shape /v1/chat/completions is unaffected. * fix(anthropic): gate total_tokens strip behind flag + handle Pydantic .usage Two P1 greptile threads on #30382: P1 — **Backwards-incompatible removal without a feature flag** Stripping `usage.total_tokens` unconditionally breaks any client currently reading the LiteLLM-shaped non-streaming /v1/messages response. Per the codebase's policy (mirrors #30418), gate behind a new flag. - `litellm.strip_anthropic_total_tokens: bool = False` (default — backward-compat: clients keep seeing total_tokens). - Env override: `LITELLM_STRIP_ANTHROPIC_TOTAL_TOKENS=true`. - Docstring: planned to flip to True in a future major release; opt in early. P1 — **Silent no-op if `result` is a Pydantic model** `base_process_llm_request` may return a Pydantic-style object whose `.usage` is a plain dict (the most common shape — e.g. objects wrapping raw upstream JSON). The original `isinstance(response, dict)` guard skipped strip on those, so `total_tokens` would still hit the wire. Helper now also reads `getattr(response, "usage", None)` and strips when that's a dict. Strongly-typed Pydantic `Usage` sub-models with required `total_tokens` fields are still skipped — those impose type constraints the helper doesn't try to subvert. Tests: - `test_strips_total_tokens_on_pydantic_model_with_dict_usage` - `test_flag_defaults_off` 8/8 pass locally. * fix(anthropic): drop env var for strip flag (docs CI) Mirrors #30418's pattern (`expose_router_debug_in_errors: bool = True`, no `os.getenv`). The `LITELLM_STRIP_ANTHROPIC_TOTAL_TOKENS` env var introduced in the prior commit was flagged by `tests/documentation_tests/test_env_keys.py` because the documentation file `docs/my-website/docs/proxy/config_settings.md` lives in `BerriAI/litellm-docs` (separate repo) and registering a new env key requires a parallel docs PR — a friction we avoid here by exposing the flag only as a Python attribute + `litellm_settings` config key, both of which load through the existing proxy config plumbing without needing the env-var registry to be updated. No semantic change: default still False, behavior identical when set via `litellm.strip_anthropic_total_tokens = True` or `litellm_settings.strip_anthropic_total_tokens: true` in config.yaml. Verified locally: env scan no longer surfaces the key; 8/8 tests pass. * ci: retrigger workflows after base branch change to litellm_internal_staging --- litellm/__init__.py | 11 +++ .../proxy/anthropic_endpoints/endpoints.py | 47 ++++++++++ .../anthropic_endpoints/test_endpoints.py | 93 +++++++++++++++++++ 3 files changed, 151 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 8daf54d1c4b..9462cfa961f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -235,6 +235,17 @@ modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API +# When True, strip the OpenAI-flavored `usage.total_tokens` field that +# LiteLLM injects into non-streaming /v1/messages responses, bringing the +# wire response into line with the Anthropic spec (matches the streaming +# SSE path, which already omits total_tokens). Default False to preserve +# backward compatibility for clients that read the LiteLLM-shaped +# `usage.total_tokens` today. Planned to flip to True in a future major +# release; opt in early via Python: +# `litellm.strip_anthropic_total_tokens = True` +# Or via `litellm_settings.strip_anthropic_total_tokens: true` in +# config.yaml. +strip_anthropic_total_tokens: bool = False route_all_chat_openai_to_responses: bool = ( os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1995ff275c9..856b788b54b 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -5,6 +5,7 @@ Unified /v1/messages endpoint - (Anthropic Spec) from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse +import litellm from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping from litellm.integrations.custom_guardrail import ModifyResponseException @@ -23,6 +24,40 @@ from litellm.types.utils import TokenCountResponse router = APIRouter() +def _strip_total_tokens_from_anthropic_response(response: Any) -> None: + """Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM + injects into Anthropic /v1/messages responses. + + The Anthropic /v1/messages spec only defines: + input_tokens, output_tokens, cache_creation_input_tokens, + cache_read_input_tokens, cache_creation.{ephemeral_5m,ephemeral_1h} + The streaming SSE path (message_delta.usage) already does not include + total_tokens; this brings the non-streaming path into the same shape. + + Handles both shapes returned by `base_process_llm_request`: + - plain `dict` (most common — `AnthropicMessagesResponse` is a TypedDict + and is `dict` at runtime) + - Pydantic model whose `usage` attribute is dict-shaped (e.g. a + BaseModel that holds raw Anthropic usage as a `dict[str, int]`) + + Streaming results (StreamingResponse, AsyncIterator, etc.) and Pydantic + models with strongly-typed Usage sub-models are left untouched — + those paths either have separate serialization handling or impose + type constraints the helper does not try to subvert. + """ + if response is None: + return + if isinstance(response, dict): + usage = response.get("usage") + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) + return + # Pydantic-model fallback: only mutate if `usage` is a dict. + usage = getattr(response, "usage", None) + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) + + @router.post( "/v1/messages", tags=["[beta] Anthropic `/v1/messages`"], @@ -72,6 +107,18 @@ async def anthropic_response( user_api_base=user_api_base, version=version, ) + # Optionally strip the non-Anthropic `usage.total_tokens` field + # LiteLLM adds internally. Anthropic's official /v1/messages spec + # only defines input_tokens / output_tokens / cache_*_input_tokens; + # total_tokens is an OpenAI convention. Default off + # (`litellm.strip_anthropic_total_tokens = False`) to preserve + # backward compatibility for clients that currently read it; set + # to True to align the wire response with the spec (and with the + # streaming SSE path, which already omits total_tokens). + # spend_logs / Prometheus still compute total internally — this + # only affects the wire response. + if litellm.strip_anthropic_total_tokens: + _strip_total_tokens_from_anthropic_response(result) return result except ModifyResponseException as e: # Guardrail flagged content in passthrough mode - return 200 with violation message diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index f6189382d74..a4da4587b7f 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -86,3 +86,96 @@ class TestEventLoggingBatchEndpoint: assert response.status_code == 200 assert response.json() == {"status": "ok"} + + +class TestStripTotalTokens(unittest.TestCase): + """Cover ``_strip_total_tokens_from_anthropic_response``. + + The Anthropic /v1/messages spec does not define ``usage.total_tokens``. + LiteLLM injects it internally; the helper must remove it from the wire + response so the non-streaming path matches the streaming SSE shape and + direct Anthropic API responses. + """ + + def test_strips_total_tokens_when_present(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = { + "id": "msg_123", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + }, + } + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response["usage"] + assert response["usage"]["input_tokens"] == 100 + assert response["usage"]["output_tokens"] == 50 + assert response["usage"]["cache_read_input_tokens"] == 0 + + def test_no_op_when_total_tokens_absent(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"usage": {"input_tokens": 100, "output_tokens": 50}} + _strip_total_tokens_from_anthropic_response(response) + assert response["usage"] == {"input_tokens": 100, "output_tokens": 50} + + def test_no_op_when_usage_missing(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"id": "msg_123"} + _strip_total_tokens_from_anthropic_response(response) + assert response == {"id": "msg_123"} + + def test_no_op_on_non_dict_response(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # Streaming responses (StreamingResponse, async iterators) are not dicts. + # The helper must not raise or attempt to mutate them. + for value in (None, "stream", 42, [{"usage": {"total_tokens": 1}}]): + _strip_total_tokens_from_anthropic_response(value) # no raise + + def test_strips_total_tokens_on_pydantic_model_with_dict_usage(self): + """Greptile P1 on #30382: helper must not silently no-op when the + response is a Pydantic-shaped object whose `usage` attribute is a + plain dict (the common case for objects wrapping raw upstream JSON). + """ + from types import SimpleNamespace + + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # SimpleNamespace mimics the .usage attribute access pattern; the + # helper's contract: if .usage is dict-shaped, strip total_tokens. + response = SimpleNamespace( + usage={"input_tokens": 100, "output_tokens": 50, "total_tokens": 150} + ) + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response.usage + assert response.usage == {"input_tokens": 100, "output_tokens": 50} + + +class TestStripTotalTokensFeatureFlag(unittest.TestCase): + """The strip is gated behind `litellm.strip_anthropic_total_tokens`. + + Default off (backward compat). Greptile P1 on #30382 required a + user-controlled flag so existing clients reading the LiteLLM-shaped + `usage.total_tokens` continue to work after this PR lands. + """ + + def test_flag_defaults_off(self): + import litellm + + assert litellm.strip_anthropic_total_tokens is False