From 381c22c9f0b08d7d747959909373588cfae0a2ff Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:22:36 +0000 Subject: [PATCH] fix: preserve hidden parameter models when stamping response costs Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../llm_response_utils/response_metadata.py | 5 ++--- .../messages/native/streaming_iterator.py | 6 +++--- .../test_response_metadata.py | 17 ++++++++++++++++- .../messages/native/test_streaming_iterator.py | 8 ++++++++ tests/unit/llms/openai/test_openai.py | 7 ++++++- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 9fd4b277a5a..b5655f579e9 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -7,6 +7,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING +from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject @@ -18,8 +19,6 @@ from litellm.types.utils import ( TranscriptionResponse, ) -_HIDDEN_PARAMS: Final = TypeAdapter(Mapping[str, object]) - def _timing_window_start( start_time: datetime.datetime, logging_obj: LiteLLMLoggingObject @@ -273,7 +272,7 @@ class ResponseMetadata: """Apply metadata to the response object""" if hasattr(self.result, "_hidden_params"): self.result._hidden_params = self._hidden_params - hidden: Final = _HIDDEN_PARAMS.validate_python(getattr(self, "_hidden_params", {})) + hidden: Final = get_hidden_params_dict(self.result) stamp_usage_cost(self.result, validated_usage_cost(hidden.get("response_cost"))) diff --git a/litellm/llms/anthropic/messages/native/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py index 3c00b3cc216..54b1dd760f3 100644 --- a/litellm/llms/anthropic/messages/native/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -470,9 +470,9 @@ class UsageCostMessagesStream: def close(self) -> None: self.sync_frames.close() - close: Final = getattr(self.stream, "close", None) - if callable(close): - close() + close_upstream: Final = getattr(self.stream, "close", None) + if callable(close_upstream): + close_upstream() def __getattr__(self, name: str) -> object: return getattr(self.stream, name) diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 63aff5a6c38..71971a56de9 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -28,7 +28,22 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse -from litellm.types.utils import ModelResponse, Usage +from litellm.types.utils import EmbeddingResponse, HiddenParams, ModelResponse, TextCompletionResponse, Usage + + +@pytest.mark.parametrize("response_type", [EmbeddingResponse, TextCompletionResponse]) +def test_metadata_apply_preserves_model_hidden_params( + response_type: type[EmbeddingResponse] | type[TextCompletionResponse], +) -> None: + response: Final = response_type() + hidden: Final = HiddenParams(response_cost=0.125) + response._hidden_params = hidden + before: Final = response.model_dump() + + ResponseMetadata(response).apply() + + assert response._hidden_params is hidden + assert response.model_dump() == before @pytest.mark.parametrize("shape", ["chat", "responses", "messages"]) diff --git a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py index 691879f166c..203b24b7778 100644 --- a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py @@ -2,6 +2,7 @@ import asyncio import json from collections.abc import AsyncIterator, Mapping, Sequence from datetime import datetime +from io import BytesIO from typing import Final, Literal from unittest.mock import patch @@ -71,6 +72,13 @@ def test_usage_cost_does_not_add_frames_to_an_empty_messages_stream() -> None: assert tuple(UsageCostMessagesStream(iter(()), _usage_cost_logging_obj())) == () +def test_usage_cost_stream_close_closes_the_upstream() -> None: + upstream: Final = BytesIO() + stream: Final = UsageCostMessagesStream(upstream, _usage_cost_logging_obj()) + stream.close() + assert upstream.closed + + class _RecordingLoggingIterator(BaseAnthropicMessagesStreamingIterator): def __init__(self, litellm_logging_obj: LiteLLMLoggingObj, request_body: dict): super().__init__(litellm_logging_obj=litellm_logging_obj, request_body=request_body) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 53122246de4..8882c964fab 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -714,7 +714,12 @@ def test_completion_defaults_missing_message_role_and_preserves_omitted_fields() "created": 1, "model": "gpt-5.4", "object": "chat.completion", - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + "cost": response._hidden_params["response_cost"], + }, }