diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 979caa29832..8d23526b41a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -197,7 +197,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool return not _supports_audio_input(model, litellm_params) -def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]: +def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]: return MappingProxyType( { key: value @@ -1455,7 +1455,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) model_response.id = _upstream_response_id(raw_response.id) or raw_response.id - for key, value in _provider_metadata(raw_response.model_extra).items(): + for key, value in responses_provider_metadata(raw_response.model_extra).items(): setattr(model_response, key, value) # Preserve hidden params from the ResponsesAPIResponse, especially the headers @@ -2139,7 +2139,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): from litellm.responses.utils import ResponseAPILoggingUtils usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage")) - provider_metadata: Final = _provider_metadata(response_data) + provider_metadata: Final = responses_provider_metadata(response_data) served_service_tier: Final = response_data.get("service_tier") return ModelResponseStream( choices=[ diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 241108b4c2f..35decbb4bd7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4352,6 +4352,9 @@ class Logging(LiteLLMLoggingBaseClass): row is built from this response rather than re-parsed from the served body.""" self.bridged_model_response = response + def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None: + self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response)) + def billable_result(self, result: object) -> object: """The response the cost calculator prices: a /v1/messages body served through the chat bridge is replaced by the ModelResponse its spend row is built from, anything else is @@ -4452,6 +4455,7 @@ class Logging(LiteLLMLoggingBaseClass): """ from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, + responses_provider_metadata, ) served_id: Final = _provider_response_id(result) @@ -4476,7 +4480,9 @@ class Logging(LiteLLMLoggingBaseClass): "usage-only ModelResponse to keep the spend_logs row.", str(e), ) - model_response: Final = litellm.ModelResponse(id=served_id) + model_response: Final = litellm.ModelResponse( + id=served_id, **responses_provider_metadata(result.model_extra) + ) model_response.model = self.model usage: Final = result.usage if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage): diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 18ecff990d2..692e5d4c175 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -43,6 +43,12 @@ _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]]) _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() +def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None: + logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs) + if logging_obj is not None: + logging_obj.record_bridged_responses_api_response(result) + + def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]: """The litellm-specific kwargs forwarded verbatim onto the Responses API request.""" return extra_kwargs or {} @@ -301,7 +307,9 @@ class LiteLLMMessagesToResponsesAPIHandler: if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - return await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator) + served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator) + _record_bridged_response(responses_kwargs, result) + return served @staticmethod def anthropic_messages_handler( diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 7f2653cb481..31eaf12a186 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6137,6 +6137,42 @@ def test_handle_anthropic_messages_response_logging_degrades_on_unparseable_resp assert result.usage.prompt_tokens == 4 # type: ignore[attr-defined] +@pytest.mark.parametrize( + "output", + [[], [{"type": "reasoning", "id": "rs_1", "status": "completed", "summary": []}]], + ids=["empty", "reasoning-only"], +) +def test_degraded_responses_payload_keeps_the_served_service_tier_price( + output: list[dict[str, object]], monkeypatch: pytest.MonkeyPatch +): + from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse + + rates: Final = { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "input_cost_per_token_flex": 5e-07, + "output_cost_per_token_flex": 2e-06, + } + monkeypatch.setitem( + litellm.model_cost, "flex-fallback-model", {"litellm_provider": "openai", "mode": "responses", **rates} + ) + degraded: Final = _anthropic_messages_logging_obj()._handle_anthropic_messages_response_logging( + result=ResponsesAPIResponse( + id="resp-flex", + created_at=1700000000, + status="completed", + output=output, + service_tier="flex", + usage=ResponseAPIUsage(input_tokens=100, output_tokens=50, total_tokens=150), + ) + ) + + assert degraded.choices == [] or degraded.choices[0].message.content in (None, "") # type: ignore[union-attr] + assert litellm.completion_cost(completion_response=degraded, model="flex-fallback-model") == pytest.approx( + 100 * rates["input_cost_per_token_flex"] + 50 * rates["output_cost_per_token_flex"] + ) + + class _SuccessCapturingLogger(CustomLogger): """Records the success payload. success_payload is populated only in async_log_success_event, so it stays None when the buggy no-op diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 61efc65f8c2..453429df084 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -553,3 +553,83 @@ def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_rea ) assert marked == unmarked + + +REASONING_MODEL: Final = "gpt-bridge-reasoning-cost" +REASONING_RATES: Final = { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "output_cost_per_reasoning_token": 3e-05, +} +REASONING_RESPONSE: Final = { + "id": "resp_bridge_reasoning", + "object": "response", + "created_at": 1, + "status": "completed", + "model": REASONING_MODEL, + "output": [ + {"type": "reasoning", "id": "rs_1", "status": "completed", "summary": []}, + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "answer", "annotations": []}], + }, + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 1240, + "output_tokens": 4040, + "total_tokens": 5280, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 3480}, + }, +} + + +@pytest.mark.asyncio +async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_tokens( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setenv("OPENAI_API_KEY", "sk-bridge-reasoning-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setitem( + litellm.model_cost, REASONING_MODEL, {"litellm_provider": "openai", "mode": "responses", **REASONING_RATES} + ) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").respond(status_code=200, json=REASONING_RESPONSE) + logging_obj: Final = Logging( + model=REASONING_MODEL, + messages=MESSAGES, + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(datetime.timezone.utc), + litellm_call_id="bridge-reasoning-0000-4000-8000-000000000001", + function_id="1234", + ) + + served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=412, + messages=MESSAGES, + model=f"openai/{REASONING_MODEL}", + custom_llm_provider="openai", + litellm_logging_obj=logging_obj, + ) + assert isinstance(served, dict) and "reasoning" not in json.dumps(served["usage"]) + + cost: Final = ProxyBaseLLMRequestProcessing._response_cost_from_logging_obj( + response=dict(served), logging_obj=logging_obj + ) + + expected_reasoning: Final = 3480 * REASONING_RATES["output_cost_per_reasoning_token"] + expected_output: Final = expected_reasoning + (4040 - 3480) * REASONING_RATES["output_cost_per_token"] + assert cost == pytest.approx(1240 * REASONING_RATES["input_cost_per_token"] + expected_output) + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) + assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)