Merge pull request #36593 from BerriAI/devin_ai_lit_5445_perplexity_stream_dict_cost

fix(streaming): accept provider cost objects when propagating usage cost

litellm/litellm_core_utils/streaming_handler.py

@@ -1830,6 +1830,20 @@ class CustomStreamWrapper:
             return
         self.chunks.append(model_response.model_copy(update={"choices": []}))
 
+    @staticmethod
+    def _resolve_provider_reported_cost(usage_cost: object) -> float | None:
+        """
+        Providers report usage.cost either as a number or, for Perplexity, as a
+        breakdown object whose total lives under ``total_cost``.
+        """
+        if isinstance(usage_cost, bool):
+            return None
+        if isinstance(usage_cost, (int, float)):
+            return float(usage_cost)
+        if isinstance(usage_cost, dict):
+            return CustomStreamWrapper._resolve_provider_reported_cost(usage_cost.get("total_cost"))
+        return None
+
     @staticmethod
     def _propagate_usage_cost_to_hidden_params(
         response: "ModelResponse",
@@ -1840,10 +1854,11 @@ class CustomStreamWrapper:
         calculator uses it instead of a token-based estimate.
         """
         _usage: Final[Usage | None] = getattr(response, "usage", None)
-        if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None:
+        _cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None))
+        if _cost is not None:
             if "additional_headers" not in response._hidden_params:
                 response._hidden_params["additional_headers"] = {}
-            response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(_usage.cost)
+            response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = _cost
 
     def __next__(self) -> "ModelResponseStream":
         cache_hit = False

tests/test_litellm/litellm_core_utils/test_streaming_handler.py

@@ -1774,6 +1774,80 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params():
     assert provider_cost == 0.00025
 
 
+def test_perplexity_streaming_dict_cost_propagates_to_hidden_params():
+    """
+    Regression: Perplexity reports usage.cost as a breakdown object, which used to
+    blow up the end of the stream with
+    `float() argument must be a string or a real number, not 'dict'`.
+    """
+    import litellm
+    from litellm.cost_calculator import get_response_cost_from_hidden_params
+
+    chunks = [
+        ModelResponseStream(
+            id="chatcmpl-pplx",
+            created=1742056047,
+            model="perplexity/sonar",
+            choices=[
+                StreamingChoices(
+                    finish_reason=None,
+                    index=0,
+                    delta=Delta(content="Hi", role="assistant"),
+                )
+            ],
+            usage=None,
+        ),
+        ModelResponseStream(
+            id="chatcmpl-pplx",
+            created=1742056048,
+            model="perplexity/sonar",
+            choices=[
+                StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))
+            ],
+            usage=None,
+        ),
+        ModelResponseStream(
+            id="chatcmpl-pplx",
+            created=1742056049,
+            model="perplexity/sonar",
+            choices=[
+                StreamingChoices(finish_reason=None, index=0, delta=Delta(content=""))
+            ],
+            usage=Usage(
+                completion_tokens=18,
+                prompt_tokens=12,
+                total_tokens=30,
+                cost={
+                    "input_tokens_cost": 0.000012,
+                    "output_tokens_cost": 0.000018,
+                    "request_cost": 0.005,
+                    "total_cost": 0.00503,
+                },
+            ),
+        ),
+    ]
+
+    complete_response = litellm.stream_chunk_builder(
+        chunks=chunks, messages=[{"role": "user", "content": "test"}]
+    )
+
+    assert complete_response is not None
+
+    CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response)
+
+    assert (
+        get_response_cost_from_hidden_params(complete_response._hidden_params)
+        == 0.00503
+    )
+
+
+def test_provider_reported_cost_ignores_unusable_shapes():
+    assert CustomStreamWrapper._resolve_provider_reported_cost(None) is None
+    assert CustomStreamWrapper._resolve_provider_reported_cost({}) is None
+    assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": None}) is None
+    assert CustomStreamWrapper._resolve_provider_reported_cost(0.5) == 0.5
+
+
 def test_handle_special_delta_attributes(
     initialized_custom_stream_wrapper: CustomStreamWrapper,
 ):