diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index c3461c849dc..99394c48792 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -11,6 +11,7 @@ from litellm.integrations._types.open_inference import ( SpanAttributes, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import is_recording from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( OTEL_SEMCONV_STABILITY_OPT_IN_ENV, OTELGenAISemconvMixin, @@ -1230,9 +1231,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): from opentelemetry.trace import Status, StatusCode span = None - # Only set attributes if the span is still recording (not closed) - # Note: parent_span is guaranteed to be not None here - if hasattr(parent_span, "set_status"): + if is_recording(parent_span): parent_span.set_status(Status(StatusCode.OK)) self.set_attributes(parent_span, kwargs, response_obj) # Raw-request as direct child of parent_span @@ -2547,7 +2546,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): def safe_set_attribute(self, span: Span, key: str, value: Any): """ Safely sets an attribute on the span, ensuring the value is a primitive type. + + Writes to a span that already ended are dropped by the OTel SDK with a + warning per attribute, so skip them. """ + if not is_recording(span): + return primitive_value: Final = self._cast_as_primitive_value_type(value) span.set_attribute(key, primitive_value) diff --git a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py index 32cf93b7937..616858cfbee 100644 --- a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py +++ b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py @@ -29,9 +29,24 @@ def cast_as_primitive_value_type(value) -> str | bool | int | float: return "" +def is_recording(span: "Span | None") -> bool: + """ + Whether ``span`` is a span that still accepts writes. + + Caller-owned parent spans (a web framework's server span, for instance) + routinely end before LiteLLM's async logging handlers run, and the OTel SDK + drops every write to an ended span with a warning. + """ + if span is None or not hasattr(span, "is_recording"): + return False + return bool(span.is_recording()) + + def safe_set_attribute(span: "Span", key: str, value: Any): """ Sets a span attribute safely with OTEL-compliant primitive typing for Arize/Phoenix. """ + if not is_recording(span): + return primitive_value: Final = cast_as_primitive_value_type(value) span.set_attribute(key, primitive_value) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index b300c386326..99fafa73fd9 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -6007,3 +6007,63 @@ class TestOTELServiceTierAttributes(unittest.TestCase): response_obj, ) self.assertEqual(attributes[self.RESPONSE_KEY], "tier-added-by-provider-later") + + +class TestEndedParentSpanNotWritten(unittest.TestCase): + """A caller-owned parent span (e.g. a framework server span) can end before + LiteLLM's async success handler runs; writing to it is dropped by the SDK and + logs a warning per attribute.""" + + HERE = os.path.dirname(__file__) + + def _load_captured(self): + with open(os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json")) as f: + kwargs = json.load(f) + with open(os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json")) as f: + response_obj = json.load(f) + return kwargs, response_obj + + @patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}, clear=False) + def test_no_writes_to_ended_ambient_server_span(self): + from opentelemetry import context as otel_context + + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + otel = OpenTelemetry(tracer_provider=tracer_provider) + kwargs, response_obj = self._load_captured() + kwargs["litellm_params"]["metadata"].pop("litellm_parent_otel_span", None) + kwargs["litellm_params"]["proxy_server_request"]["headers"].pop("traceparent", None) + + server_span = tracer_provider.get_tracer(__name__).start_span(name="POST /chat") + token = otel_context.attach(trace.set_span_in_context(server_span)) + try: + server_span.end() + start = datetime.utcnow() + end = start + timedelta(seconds=1) + with self.assertNoLogs("opentelemetry.sdk.trace", level="WARNING"): + otel._handle_success(kwargs, response_obj, start, end) + finally: + otel_context.detach(token) + + exported_server_spans = [s for s in span_exporter.get_finished_spans() if s.name == "POST /chat"] + self.assertEqual(len(exported_server_spans), 1) + self.assertNotIn("gen_ai.request.model", exported_server_spans[0].attributes or {}) + + def test_safe_set_attribute_skips_ended_span(self): + tracer_provider = TracerProvider() + span = tracer_provider.get_tracer(__name__).start_span(name="ended") + span.end() + + with self.assertNoLogs("opentelemetry.sdk.trace", level="WARNING"): + OpenTelemetry(tracer_provider=tracer_provider).safe_set_attribute(span, "gen_ai.request.model", "gpt-5") + + def test_safe_set_attribute_still_writes_to_recording_span(self): + tracer_provider = TracerProvider() + span = tracer_provider.get_tracer(__name__).start_span(name="recording") + + OpenTelemetry(tracer_provider=tracer_provider).safe_set_attribute(span, "gen_ai.request.model", "gpt-5") + span.end() + + self.assertEqual((span.attributes or {}).get("gen_ai.request.model"), "gpt-5")