From 7c8673f8cc856fa773619ca48724faa037dd74e0 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 2 Oct 2026 16:39:46 +0200 Subject: [PATCH 1/2] fix(otel): leave out span cost when the price lookup failed A model missing from the price map gets response_cost 0.0 in the standard logging payload, with response_cost_failure_debug_info set. The OpenInference writer (arize, phoenix, langfuse_otel) and the OTel v2 span data copied that 0.0 into llm.cost.total, litellm.cost.total and langfuse.observation.cost_details, so backends showed the call as free. Skip the cost when the lookup failed, except on cache hits, which are free. --- litellm/integrations/arize/_utils.py | 8 +++- litellm/integrations/otel/model/payloads.py | 6 ++- .../integrations/arize/test_arize_utils.py | 44 ++++++++++++++++++- .../integrations/otel/test_otel_v2_emitter.py | 33 ++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 0271cf1e03c..5c7cb99a9e5 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -891,11 +891,17 @@ def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None: single total in `StandardLoggingPayload.response_cost`, so we cannot split it into prompt/completion. We also keep the legacy `llm.response.cost` key for back-compat with any consumer querying it. + When the price lookup failed, `response_cost` is a placeholder 0.0 and + no cost is emitted, so the backend does not record the call as free. """ if not isinstance(standard_logging_payload, dict): return cost: Final = standard_logging_payload.get("response_cost") - if cost is None: + cost_lookup_failed: Final = ( + standard_logging_payload.get("response_cost_failure_debug_info") is not None + and standard_logging_payload.get("cache_hit") is not True + ) + if cost is None or cost_lookup_failed: return try: cost_value: Final = float(cost) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index e06cc1d0407..9f3bd70d5ef 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -462,6 +462,10 @@ class LLMCallSpanData: embedding_output: Final = ( EmbeddingOutput.from_response(response) if operation is GenAIOperation.EMBEDDINGS else None ) + # A failed price lookup leaves a placeholder 0.0 in response_cost, while a cache hit is really free + cost_lookup_failed: Final = ( + payload.get("response_cost_failure_debug_info") is not None and payload.get("cache_hit") is not True + ) return cls( operation=operation, provider=resolve_provider(as_str(payload.get("custom_llm_provider"))), @@ -472,7 +476,7 @@ class LLMCallSpanData: usage=LLMUsage.from_standard_logging_payload(payload), finish_reasons=finish_reasons, error=_parse_error(payload), - response_cost=as_float(payload.get("response_cost")), + response_cost=None if cost_lookup_failed else as_float(payload.get("response_cost")), cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))), server=ServerInfo.from_api_base(context.api_base), identity=context.identity, diff --git a/tests/unit/integrations/arize/test_arize_utils.py b/tests/unit/integrations/arize/test_arize_utils.py index 167b083e147..dffb6eb168f 100644 --- a/tests/unit/integrations/arize/test_arize_utils.py +++ b/tests/unit/integrations/arize/test_arize_utils.py @@ -6,6 +6,7 @@ from typing import Optional import asyncio import pytest +from opentelemetry.sdk.trace import TracerProvider import litellm from litellm.integrations._types.open_inference import ( @@ -15,7 +16,7 @@ from litellm.integrations._types.open_inference import ( ) from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_logger import CustomLogger -from litellm.types.utils import Choices, StandardCallbackDynamicParams +from litellm.types.utils import Choices, ModelResponse, StandardCallbackDynamicParams def test_arize_set_attributes(): @@ -1013,6 +1014,47 @@ def test_arize_emits_response_cost(): assert attrs["llm.response.cost"] == 0.0012345 # legacy key still emitted +_PRICE_LOOKUP_FAILED = {"error_str": "This model isn't mapped yet. model=openai/unpriced-model", "traceback_str": ""} + + +@pytest.mark.parametrize( + ("cost_fields", "expected_cost"), + [ + pytest.param({"response_cost": 0.0}, 0.0, id="free-model"), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED}, + None, + id="unpriced", + ), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED, "cache_hit": True}, + 0.0, + id="unpriced-cache-hit", + ), + ], +) +def test_arize_cost_attrs_tell_unpriced_call_from_free_call(cost_fields, expected_cost): + span = TracerProvider().get_tracer(__name__).start_span("litellm_request") + kwargs = { + "model": "unpriced-model", + "messages": [{"role": "user", "content": "hi"}], + "standard_logging_object": {"model_parameters": {}, "metadata": {}, "call_type": "completion", **cost_fields}, + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + } + response_obj = ModelResponse( + usage={"total_tokens": 30, "completion_tokens": 20, "prompt_tokens": 10}, + choices=[Choices(message={"role": "assistant", "content": "ok"})], + model="unpriced-model", + id="r4", + ) + + ArizeLogger.set_arize_attributes(span, kwargs, response_obj) + + assert span.attributes.get("llm.cost.total") == expected_cost + assert span.attributes.get("llm.response.cost") == expected_cost + + def test_arize_passthrough_bedrock_anthropic_normalization(): """Bedrock-Anthropic passthrough: input/output text must be set so the span renders something other than raw provider attrs.""" diff --git a/tests/unit/integrations/otel/test_otel_v2_emitter.py b/tests/unit/integrations/otel/test_otel_v2_emitter.py index 11b2aa5fd67..724709d8b6c 100644 --- a/tests/unit/integrations/otel/test_otel_v2_emitter.py +++ b/tests/unit/integrations/otel/test_otel_v2_emitter.py @@ -22,6 +22,7 @@ from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 from litellm.integrations.otel.emitter import SpanEmitter, span_attribute_limit # noqa: E402 from litellm.integrations.otel.emitter import stamp_error # noqa: E402 +from litellm.integrations.otel.mappers import LangfuseMapper # noqa: E402 from litellm.integrations.otel.mappers.utils import MAX_TOOL_DEFINITION_ATTRS_PER_SPAN # noqa: E402 from litellm.integrations.otel.model.payloads import ( # noqa: E402 GuardrailSpanData, @@ -90,6 +91,38 @@ def test_llm_call_span_cost_breakdown(): assert f"{LiteLLM.COST_PREFIX}margin_total_amount" not in a +_PRICE_LOOKUP_FAILED = {"error_str": "This model isn't mapped yet. model=openai/unpriced-model", "traceback_str": ""} + + +@pytest.mark.parametrize( + ("cost_fields", "expected_cost"), + [ + pytest.param({"response_cost": 0.0}, 0.0, id="free-model"), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED}, + None, + id="unpriced", + ), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED, "cache_hit": True}, + 0.0, + id="unpriced-cache-hit", + ), + ], +) +def test_llm_call_span_tells_unpriced_call_from_free_call(cost_fields, expected_cost): + engine, exporter = _engine() + data = LLMCallSpanData.from_standard_logging_payload(_payload(**cost_fields)) + + engine.emit(SpanRole.LLM_CALL, data) + + (span,) = exporter.get_finished_spans() + assert span.attributes.get(f"{LiteLLM.COST_PREFIX}total") == expected_cost + langfuse_cost_details = LangfuseMapper().map(data).get("langfuse.observation.cost_details") + langfuse_cost = json.loads(langfuse_cost_details)["total"] if langfuse_cost_details is not None else None + assert langfuse_cost == expected_cost + + def test_tracer_scope_carries_litellm_version(): from litellm._version import version as litellm_version From 43b96d653d44e90ae6bca056298bba95e95ebead Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Sat, 3 Oct 2026 12:07:58 +0200 Subject: [PATCH 2/2] test(otel): keep the guardrail charge on the guardrail span for unpriced calls --- .../integrations/otel/test_otel_v2_emitter.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit/integrations/otel/test_otel_v2_emitter.py b/tests/unit/integrations/otel/test_otel_v2_emitter.py index 724709d8b6c..5537118f796 100644 --- a/tests/unit/integrations/otel/test_otel_v2_emitter.py +++ b/tests/unit/integrations/otel/test_otel_v2_emitter.py @@ -123,6 +123,27 @@ def test_llm_call_span_tells_unpriced_call_from_free_call(cost_fields, expected_ assert langfuse_cost == expected_cost +def test_unpriced_call_keeps_the_guardrail_charge_on_the_guardrail_span(): + engine, exporter = _engine() + payload = _payload( + response_cost=0.002, + response_cost_failure_debug_info=_PRICE_LOOKUP_FAILED, + cost_breakdown={"guardrail_cost": 0.002, "total_cost": 0.002}, + ) + + engine.emit(SpanRole.LLM_CALL, LLMCallSpanData.from_standard_logging_payload(payload)) + engine.emit( + SpanRole.GUARDRAIL, + GuardrailSpanData.from_logging_entry( + {"guardrail_name": "bedrock", "guardrail_status": "success", "guardrail_cost": 0.002} + ), + ) + + llm_span, guardrail_span = exporter.get_finished_spans() + assert f"{LiteLLM.COST_PREFIX}total" not in llm_span.attributes + assert guardrail_span.attributes[LiteLLM.GUARDRAIL_COST] == 0.002 + + def test_tracer_scope_carries_litellm_version(): from litellm._version import version as litellm_version