From 50ef3da304027663e69effc678ba843070e13d87 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 26 Aug 2026 00:18:49 -0700 Subject: [PATCH] fix(otel): stop the legacy emitter reporting replayed tokens on response reads The zeroing so far lands in the standard logging payload, which the legacy OpenTelemetry emitter does not read for usage: it takes prompt, completion and total tokens straight off the response object, so a retrieval span still carried the token counts of the call that produced the response, and the token usage histogram still recorded them. That emitter is the default, so the spend row said zero while the trace said otherwise. The background cost poller keeps its counts, the same exemption the pricing path already makes. --- litellm/integrations/opentelemetry.py | 29 ++++++++- .../integrations/test_opentelemetry.py | 60 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 78081837ae3..62aa2c24e74 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, TypedDict, cast import litellm from litellm._logging import verbose_logger +from litellm.constants import NON_INFERENCE_CALL_TYPES from litellm.integrations._types.open_inference import ( OpenInferenceSpanKindValues, SpanAttributes, @@ -22,6 +23,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( ) from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.semconv import Metric +from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.service_tier_utils import ( @@ -238,6 +240,20 @@ def _freeze_for_dedupe(value: object, _depth: int = 0) -> HashableScope: return repr(value) +def _is_unbilled_non_inference(call_type: object, litellm_params: Mapping[str, Any]) -> bool: + """Whether this call is a read or management route whose token counts describe an + earlier request rather than this one. + + The call-type membership test runs first so that inference traffic, which is every + request in a normal workload, never pays for the metadata merge behind it. + """ + if call_type not in NON_INFERENCE_CALL_TYPES: + return False + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + return is_unbilled_non_inference_call(call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params)) + + def _shutdown_tracer_provider(provider: "_SDKTracerProvider") -> None: """Flush and stop a dropped provider so its exporter thread is reclaimed.""" try: @@ -1643,7 +1659,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if self._operation_duration_histogram: self._operation_duration_histogram.record(duration_s, attributes=common_attrs) - if response_obj and (usage := response_obj.get("usage")) and self._token_usage_histogram: + if ( + response_obj + and not _is_unbilled_non_inference(kwargs.get("call_type"), params) + and (usage := response_obj.get("usage")) + and self._token_usage_histogram + ): in_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"} out_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"} self._token_usage_histogram.record(usage.get("prompt_tokens", 0), attributes=in_attrs) @@ -2468,7 +2489,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload) - usage: Final = response_obj and response_obj.get("usage") + usage: Final = ( + response_obj.get("usage") + if response_obj and not _is_unbilled_non_inference(kwargs.get("call_type"), litellm_params) + else None + ) if usage: self.safe_set_attribute( span=span, diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 229214bf1e1..e0cdd7519ff 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -6345,3 +6345,63 @@ class TestOpenTelemetryDatabaseSemconvAttributes(unittest.TestCase): span = self._service_span(ServiceTypes.DB, "get_data", None) self.assertEqual(span.attributes["db.system.name"], "postgresql") self.assertNotIn("server.address", span.attributes) + + +class TestOpenTelemetryNonInferenceUsage(unittest.TestCase): + """Reading a stored response replays the usage of the call that created it, so emitting those + token counts again on the read's span reports the same tokens a second time. Regression tests + for LIT-5602, covering the legacy emitter that runs by default.""" + + USAGE = {"prompt_tokens": 4000, "completion_tokens": 2000, "total_tokens": 6000} + TOKEN_KEYS = frozenset({"gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens", "gen_ai.usage.total_tokens"}) + BACKGROUND_POLL = {"internal_call_origin": "background_response_cost_poll"} + RESPONSE_OBJ = {"id": "resp_lit5602", "model": "gpt-4o", "usage": USAGE} + + def _kwargs(self, call_type, litellm_metadata=None): + return { + "model": "gpt-4o", + "call_type": call_type, + "optional_params": {}, + "litellm_params": { + "custom_llm_provider": "openai", + "litellm_metadata": litellm_metadata or {}, + }, + "standard_logging_object": {"id": "lit5602", "call_type": call_type, "metadata": {}}, + } + + def _token_attributes_on_span(self, call_type, litellm_metadata=None): + otel = OpenTelemetry() + mock_span = MagicMock() + otel.set_attributes( + span=mock_span, + kwargs=self._kwargs(call_type, litellm_metadata), + response_obj=dict(self.RESPONSE_OBJ), + ) + return {call[0][0] for call in mock_span.set_attribute.call_args_list if call[0][0] in self.TOKEN_KEYS} + + def _token_histogram_calls(self, call_type, litellm_metadata=None): + otel = OpenTelemetry() + otel._operation_duration_histogram = MagicMock() + otel._token_usage_histogram = MagicMock() + otel._cost_histogram = None + now = datetime.now() + otel._record_metrics(self._kwargs(call_type, litellm_metadata), dict(self.RESPONSE_OBJ), now, now) + return otel._token_usage_histogram.record.call_count + + def test_inference_call_still_reports_its_tokens_on_the_span(self): + self.assertEqual(self._token_attributes_on_span("acompletion"), set(self.TOKEN_KEYS)) + + def test_response_read_does_not_report_the_retrieved_tokens_on_the_span(self): + self.assertEqual(self._token_attributes_on_span("aget_responses"), set()) + + def test_background_cost_poll_read_still_reports_its_tokens_on_the_span(self): + self.assertEqual(self._token_attributes_on_span("aget_responses", self.BACKGROUND_POLL), set(self.TOKEN_KEYS)) + + def test_inference_call_still_records_the_token_usage_histogram(self): + self.assertEqual(self._token_histogram_calls("acompletion"), 2) + + def test_response_read_does_not_record_the_token_usage_histogram(self): + self.assertEqual(self._token_histogram_calls("aget_responses"), 0) + + def test_background_cost_poll_read_still_records_the_token_usage_histogram(self): + self.assertEqual(self._token_histogram_calls("aget_responses", self.BACKGROUND_POLL), 2)