From e5c74cb2a68c11671d2c42738fb8ae8c69aab629 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 30 Sep 2026 10:17:21 -0700 Subject: [PATCH] fix(cost_calculator): stop copying optional_params into response hidden params (#43637) * fix(cost_calculator): stop copying optional_params into response hidden params * test(cost_calculator): assert the stored spend-log request and logging payload carry no forwarded credentials --- litellm/cost_calculator.py | 1 - litellm/responses/streaming_iterator.py | 2 +- tests/unit/test_cost_calculator.py | 75 +++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6b3e739ac4f..238b7cc3fdd 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2008,7 +2008,6 @@ def response_cost_calculator( else: if isinstance(response_object, BaseModel): if hasattr(response_object, "_hidden_params"): - response_object._hidden_params["optional_params"] = optional_params provider_response_cost: Final = get_response_cost_from_hidden_params(response_object._hidden_params) if provider_response_cost is not None: return provider_response_cost diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 12bc9adbac8..10c73071fc7 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -601,7 +601,7 @@ class BaseResponsesAPIStreamingIterator: raw_headers: Final[Mapping[str, object]] = raw if isinstance(raw, Mapping) else EMPTY_MAPPING # rebuild by value and let existing keys win: sharing the source dicts would alias what the proxy # splats into the client's HTTP headers, and copying non-header keys would carry response_cost - target._hidden_params = { # mutable-ok: the cost calculator writes optional_params into _hidden_params + target._hidden_params = { # mutable-ok: logging aliases _hidden_params into request metadata and writes into it "additional_headers": {**headers}, # mutable-ok: fresh copy, logging callbacks may mutate it "headers": {**raw_headers}, # mutable-ok: fresh copy, logging callbacks may mutate it **existing, diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 2fe09e6de98..36e188e82d6 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -26,6 +26,7 @@ from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, Choices, + EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, @@ -160,6 +161,80 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 +def test_response_cost_calculator_keeps_optional_params_out_of_hidden_params(): + class MockResponse(BaseModel): + pass + + response = MockResponse() + response._hidden_params = {"custom_llm_provider": "openai"} + optional_params = { + "dimensions": 256, + "extra_headers": {"x-goog-api-key": "goog-secret"}, + "aws_session_token": "session-secret", + } + + response_cost_calculator( + response_object=response, + model="text-embedding-3-small", + custom_llm_provider="openai", + call_type="embedding", + optional_params=optional_params, + ) + + assert response._hidden_params == {"custom_llm_provider": "openai"} + assert optional_params["extra_headers"] == {"x-goog-api-key": "goog-secret"} + assert optional_params["aws_session_token"] == "session-secret" + + +def test_embedding_success_logging_and_spend_log_carry_no_forwarded_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_proxy_server_request_for_spend_logs_payload + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + shared_metadata: dict[str, object] = {"user_api_key_alias": "alias"} + proxy_server_request: Final = {"body": {"model": "emb", "input": "hi", "metadata": shared_metadata}} + shared_optional_params: dict[str, object] = {"encoding_format": "float"} + logging_obj = Logging( + model="text-embedding-3-small", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="aembedding", + start_time=datetime.datetime.now(), + litellm_call_id="embedding-hidden-params", + function_id="f", + ) + logging_obj.update_environment_variables( + model="text-embedding-3-small", + litellm_params={"metadata": shared_metadata, "proxy_server_request": proxy_server_request}, + optional_params=shared_optional_params, + custom_llm_provider="openai", + ) + shared_optional_params["extra_headers"] = {"x-goog-api-key": "goog-secret"} + response = EmbeddingResponse(model="text-embedding-3-small", data=[], usage=Usage(prompt_tokens=3, total_tokens=3)) + response._hidden_params = {"custom_llm_provider": "openai"} + + logging_obj._process_hidden_params_and_response_cost( + response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + + litellm_params = logging_obj.model_call_details["litellm_params"] + stored_request: Final = _get_proxy_server_request_for_spend_logs_payload( + metadata=shared_metadata, + litellm_params=litellm_params, + kwargs=logging_obj.model_call_details, + ) + hidden_params = litellm_params["metadata"]["hidden_params"] + assert isinstance(hidden_params, dict) + assert "optional_params" not in hidden_params + assert '"hidden_params"' in stored_request + assert "goog-secret" not in stored_request + assert "goog-secret" not in str(logging_obj.model_call_details["standard_logging_object"]) + assert logging_obj.model_call_details["response_cost"] is not None + assert logging_obj.optional_params["extra_headers"] == {"x-goog-api-key": "goog-secret"} + +