Merge 43b96d653d into 6532dcb73b

litellm/integrations/arize/_utils.py

@@ -891,11 +891,17 @@ def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None:
     single total in `StandardLoggingPayload.response_cost`, so we cannot
     split it into prompt/completion. We also keep the legacy
     `llm.response.cost` key for back-compat with any consumer querying it.
+    When the price lookup failed, `response_cost` is a placeholder 0.0 and
+    no cost is emitted, so the backend does not record the call as free.
     """
     if not isinstance(standard_logging_payload, dict):
         return
     cost: Final = standard_logging_payload.get("response_cost")
-    if cost is None:
+    cost_lookup_failed: Final = (
+        standard_logging_payload.get("response_cost_failure_debug_info") is not None
+        and standard_logging_payload.get("cache_hit") is not True
+    )
+    if cost is None or cost_lookup_failed:
         return
     try:
         cost_value: Final = float(cost)

litellm/integrations/otel/model/payloads.py

@@ -469,6 +469,10 @@ class LLMCallSpanData:
         embedding_output: Final = (
             EmbeddingOutput.from_response(response) if operation is GenAIOperation.EMBEDDINGS else None
         )
+        # A failed price lookup leaves a placeholder 0.0 in response_cost, while a cache hit is really free
+        cost_lookup_failed: Final = (
+            payload.get("response_cost_failure_debug_info") is not None and payload.get("cache_hit") is not True
+        )
         return cls(
             operation=operation,
             provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
@@ -479,7 +483,7 @@ class LLMCallSpanData:
             usage=LLMUsage.from_standard_logging_payload(payload),
             finish_reasons=finish_reasons,
             error=_parse_error(payload),
-            response_cost=as_float(payload.get("response_cost")),
+            response_cost=None if cost_lookup_failed else as_float(payload.get("response_cost")),
             cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))),
             server=ServerInfo.from_api_base(context.api_base),
             identity=context.identity,

tests/unit/integrations/arize/test_arize_utils.py

@@ -6,6 +6,7 @@ from typing import Optional
 import asyncio
 
 import pytest
+from opentelemetry.sdk.trace import TracerProvider
 
 import litellm
 from litellm.integrations._types.open_inference import (
@@ -15,7 +16,7 @@ from litellm.integrations._types.open_inference import (
 )
 from litellm.integrations.arize.arize import ArizeLogger
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.types.utils import Choices, StandardCallbackDynamicParams
+from litellm.types.utils import Choices, ModelResponse, StandardCallbackDynamicParams
 
 
 def test_arize_set_attributes():
@@ -1013,6 +1014,47 @@ def test_arize_emits_response_cost():
     assert attrs["llm.response.cost"] == 0.0012345  # legacy key still emitted
 
 
+_PRICE_LOOKUP_FAILED = {"error_str": "This model isn't mapped yet. model=openai/unpriced-model", "traceback_str": ""}
+
+
+@pytest.mark.parametrize(
+    ("cost_fields", "expected_cost"),
+    [
+        pytest.param({"response_cost": 0.0}, 0.0, id="free-model"),
+        pytest.param(
+            {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED},
+            None,
+            id="unpriced",
+        ),
+        pytest.param(
+            {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED, "cache_hit": True},
+            0.0,
+            id="unpriced-cache-hit",
+        ),
+    ],
+)
+def test_arize_cost_attrs_tell_unpriced_call_from_free_call(cost_fields, expected_cost):
+    span = TracerProvider().get_tracer(__name__).start_span("litellm_request")
+    kwargs = {
+        "model": "unpriced-model",
+        "messages": [{"role": "user", "content": "hi"}],
+        "standard_logging_object": {"model_parameters": {}, "metadata": {}, "call_type": "completion", **cost_fields},
+        "optional_params": {},
+        "litellm_params": {"custom_llm_provider": "openai"},
+    }
+    response_obj = ModelResponse(
+        usage={"total_tokens": 30, "completion_tokens": 20, "prompt_tokens": 10},
+        choices=[Choices(message={"role": "assistant", "content": "ok"})],
+        model="unpriced-model",
+        id="r4",
+    )
+
+    ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
+
+    assert span.attributes.get("llm.cost.total") == expected_cost
+    assert span.attributes.get("llm.response.cost") == expected_cost
+
+
 def test_arize_passthrough_bedrock_anthropic_normalization():
     """Bedrock-Anthropic passthrough: input/output text must be set so the
     span renders something other than raw provider attrs."""

tests/unit/integrations/otel/test_otel_v2_emitter.py

@@ -22,6 +22,7 @@ from litellm.integrations.otel.plumbing import context as ctx_mod  # noqa: E402
 from litellm.integrations.otel.plumbing import providers  # noqa: E402
 from litellm.integrations.otel.emitter import SpanEmitter, span_attribute_limit  # noqa: E402
 from litellm.integrations.otel.emitter import stamp_error  # noqa: E402
+from litellm.integrations.otel.mappers import LangfuseMapper  # noqa: E402
 from litellm.integrations.otel.mappers.utils import MAX_TOOL_DEFINITION_ATTRS_PER_SPAN  # noqa: E402
 from litellm.integrations.otel.model.payloads import (  # noqa: E402
     GuardrailSpanData,
@@ -90,6 +91,59 @@ def test_llm_call_span_cost_breakdown():
     assert f"{LiteLLM.COST_PREFIX}margin_total_amount" not in a
 
 
+_PRICE_LOOKUP_FAILED = {"error_str": "This model isn't mapped yet. model=openai/unpriced-model", "traceback_str": ""}
+
+
+@pytest.mark.parametrize(
+    ("cost_fields", "expected_cost"),
+    [
+        pytest.param({"response_cost": 0.0}, 0.0, id="free-model"),
+        pytest.param(
+            {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED},
+            None,
+            id="unpriced",
+        ),
+        pytest.param(
+            {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED, "cache_hit": True},
+            0.0,
+            id="unpriced-cache-hit",
+        ),
+    ],
+)
+def test_llm_call_span_tells_unpriced_call_from_free_call(cost_fields, expected_cost):
+    engine, exporter = _engine()
+    data = LLMCallSpanData.from_standard_logging_payload(_payload(**cost_fields))
+
+    engine.emit(SpanRole.LLM_CALL, data)
+
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes.get(f"{LiteLLM.COST_PREFIX}total") == expected_cost
+    langfuse_cost_details = LangfuseMapper().map(data).get("langfuse.observation.cost_details")
+    langfuse_cost = json.loads(langfuse_cost_details)["total"] if langfuse_cost_details is not None else None
+    assert langfuse_cost == expected_cost
+
+
+def test_unpriced_call_keeps_the_guardrail_charge_on_the_guardrail_span():
+    engine, exporter = _engine()
+    payload = _payload(
+        response_cost=0.002,
+        response_cost_failure_debug_info=_PRICE_LOOKUP_FAILED,
+        cost_breakdown={"guardrail_cost": 0.002, "total_cost": 0.002},
+    )
+
+    engine.emit(SpanRole.LLM_CALL, LLMCallSpanData.from_standard_logging_payload(payload))
+    engine.emit(
+        SpanRole.GUARDRAIL,
+        GuardrailSpanData.from_logging_entry(
+            {"guardrail_name": "bedrock", "guardrail_status": "success", "guardrail_cost": 0.002}
+        ),
+    )
+
+    llm_span, guardrail_span = exporter.get_finished_spans()
+    assert f"{LiteLLM.COST_PREFIX}total" not in llm_span.attributes
+    assert guardrail_span.attributes[LiteLLM.GUARDRAIL_COST] == 0.002
+
+
 def test_tracer_scope_carries_litellm_version():
     from litellm._version import version as litellm_version