From c19a7f7dcb3c2cca9d4735fc73bf1de9f8d940e9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:20:16 +0000 Subject: [PATCH] refactor(anthropic): keep fast-mode speed plumbing within lint budgets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../streaming_chunk_builder_utils.py | 47 ++++++++++++------- .../anthropic_passthrough_logging_handler.py | 19 ++++---- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 5ab44d03520..c14e2db3c33 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -730,8 +730,6 @@ class ChunkProcessor: # lost and 1h cache writes get billed at the 5m rate. cache_creation_token_details: CacheCreationTokenDetails | None = None cost: float | None = None - inference_geo: str | None = None - speed: str | None = None for chunk in chunks: usage_chunk = self._extract_usage_chunk(chunk) @@ -787,13 +785,6 @@ class ChunkProcessor: if usage_chunk_dict["cost"] is not None: cost = usage_chunk_dict["cost"] - chunk_inference_geo = getattr(usage_chunk, "inference_geo", None) - if isinstance(chunk_inference_geo, str): - inference_geo = chunk_inference_geo - chunk_speed = getattr(usage_chunk, "speed", None) - if isinstance(chunk_speed, str): - speed = chunk_speed - prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details) completion_tokens = self._reset_anthropic_cursor_completion_tokens( @@ -812,10 +803,28 @@ class ChunkProcessor: completion_tokens_details=completion_tokens_details, prompt_tokens_details=prompt_tokens_details, cost=cost, - inference_geo=inference_geo, - speed=speed, + inference_geo=self._last_provider_pricing_field(chunks, "inference_geo"), + speed=self._last_provider_pricing_field(chunks, "speed"), ) + def _last_provider_pricing_field( + self, + chunks: Sequence["_UsageBearingChunk | ModelResponse"], + field: str, + ) -> str | None: + """ + Last value of a provider-specific usage field that changes pricing but is not a + declared ``Usage`` field, e.g. Anthropic's ``speed`` (fast mode multiplies + non-cache token cost) and ``inference_geo``. + """ + values: Final = [ + value + for chunk in chunks + if (usage_chunk := self._extract_usage_chunk(chunk)) is not None + and isinstance(value := getattr(usage_chunk, field, None), str) + ] + return values[-1] if values else None + @staticmethod def _reset_anthropic_cursor_completion_tokens( chunks: Sequence["_UsageBearingChunk | ModelResponse"], @@ -943,14 +952,18 @@ class ChunkProcessor: if cost is not None: setattr(returned_usage, "cost", cost) - if calculated_usage_per_chunk["inference_geo"] is not None: - setattr(returned_usage, "inference_geo", calculated_usage_per_chunk["inference_geo"]) - if calculated_usage_per_chunk["speed"] is not None: - setattr(returned_usage, "speed", calculated_usage_per_chunk["speed"]) - # Return a new usage object with the new values - returned_usage = Usage(**returned_usage.model_dump()) + provider_pricing_fields: Final = { + field: value + for field, value in ( + ("inference_geo", calculated_usage_per_chunk["inference_geo"]), + ("speed", calculated_usage_per_chunk["speed"]), + ) + if value is not None + } + + returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields) return returned_usage diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index b60edab0a66..1d8054e6cd3 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -1,5 +1,5 @@ import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, cast @@ -74,6 +74,9 @@ class AnthropicPassthroughLoggingHandler: ) model: Final = response_body.get("model", "") + speed: Final = AnthropicPassthroughLoggingHandler._cost_relevant_speed( + request_body or kwargs.get("request_body") + ) anthropic_config: Final = get_anthropic_config(url_route) litellm_model_response: Final[ModelResponse] = anthropic_config().transform_response( raw_response=httpx_response, @@ -81,9 +84,7 @@ class AnthropicPassthroughLoggingHandler: model=model, messages=[], logging_obj=logging_obj, - optional_params=AnthropicPassthroughLoggingHandler._cost_relevant_request_params( - request_body or kwargs.get("request_body") - ), + optional_params={"speed": speed} if speed else {}, api_key="", request_data={}, encoding=litellm.encoding, @@ -106,13 +107,13 @@ class AnthropicPassthroughLoggingHandler: } @staticmethod - def _cost_relevant_request_params(request_body: dict | None) -> dict: + def _cost_relevant_speed(request_body: Mapping[str, object] | None) -> str | None: """ - Request params that change how the response is priced, and so must reach the - usage-building paths. Anthropic's ``speed=fast`` multiplies non-cache token cost. + Anthropic's ``speed=fast`` multiplies non-cache token cost, and only the request + carries it, so it has to reach the usage-building paths for spend to be right. """ speed: Final = (request_body or {}).get("speed") - return {"speed": speed} if isinstance(speed, str) else {} + return speed if isinstance(speed, str) else None @staticmethod def _get_user_from_metadata( @@ -327,7 +328,7 @@ class AnthropicPassthroughLoggingHandler: - Logs in litellm callbacks """ - speed: Final = AnthropicPassthroughLoggingHandler._cost_relevant_request_params(request_body).get("speed") + speed: Final = AnthropicPassthroughLoggingHandler._cost_relevant_speed(request_body) model = request_body.get("model", "") # Check if it's available in the logging object if (