diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 2de8e8bf7f1..d6b10de7f0c 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1075,7 +1075,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): def _record_metrics(self, kwargs, response_obj, start_time, end_time): duration_s = (end_time - start_time).total_seconds() params = kwargs.get("litellm_params") or {} - provider = params.get("custom_llm_provider", "Unknown") + provider = params.get("custom_llm_provider") or "Unknown" common_attrs = { "gen_ai.operation.name": ( @@ -1122,7 +1122,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): duration_s, attributes=common_attrs ) if ( - response_obj + isinstance(response_obj, dict) and (usage := response_obj.get("usage")) and self._token_usage_histogram ): @@ -1211,7 +1211,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Get completion tokens from response_obj completion_tokens = None - if response_obj and (usage := response_obj.get("usage")): + if isinstance(response_obj, dict) and (usage := response_obj.get("usage")): completion_tokens = usage.get("completion_tokens") if completion_tokens is None or completion_tokens <= 0: @@ -1327,6 +1327,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if not self.config.enable_events: return + if not isinstance(response_obj, dict): + return + # NOTE: Semantic logs (gen_ai.content.prompt/completion events) have compatibility issues # with OTEL SDK >= 1.39.0 due to breaking changes in PR #4676: # - LogRecord moved from opentelemetry.sdk._logs to opentelemetry.sdk._logs._internal @@ -1345,8 +1348,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): parent_ctx = span.get_span_context() provider = (kwargs.get("litellm_params") or {}).get( - "custom_llm_provider", "Unknown" - ) + "custom_llm_provider" + ) or "Unknown" if self._gen_ai_semconv_latest_experimental: self._emit_inference_details_event( @@ -1369,7 +1372,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): attrs["id"] = msg["id"] capture_event_content = self._capture_in_event() if capture_event_content and msg.get("content"): - attrs["gen_ai.prompt"] = msg["content"] + content = msg["content"] + if isinstance(content, str): + attrs["gen_ai.prompt"] = content + else: + attrs["gen_ai.prompt"] = safe_dumps(content) body = msg.copy() if not capture_event_content: @@ -1398,7 +1405,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): body_msg = choice.get("message", {}) capture_event_content = self._capture_in_event() if capture_event_content and body_msg.get("content"): - attrs["message.content"] = body_msg["content"] + completion_content = body_msg["content"] + if isinstance(completion_content, str): + attrs["message.content"] = completion_content + else: + attrs["message.content"] = safe_dumps(completion_content) body = { "index": idx, "finish_reason": choice.get("finish_reason"), @@ -1876,7 +1887,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) # The Generative AI Provider: Azure, OpenAI, etc. - provider_name = litellm_params.get("custom_llm_provider", "Unknown") + provider_name = litellm_params.get("custom_llm_provider") or "Unknown" # Latest-experimental semconv replaced gen_ai.system with # gen_ai.provider.name; emit only the conformant key in that mode. if self._gen_ai_semconv_latest_experimental: @@ -1941,7 +1952,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # the litellm call ID so every call type can be correlated # across LiteLLM UI, Phoenix traces, and provider logs (Issue #8). response_id = ( - response_obj.get("id") if response_obj else None + response_obj.get("id") if isinstance(response_obj, dict) else None ) or standard_logging_payload.get("id") if response_id: self.safe_set_attribute( @@ -1959,14 +1970,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) # The model used to generate the response. - if response_obj and response_obj.get("model"): + if isinstance(response_obj, dict) and response_obj.get("model"): self.safe_set_attribute( span=span, key=SpanAttributes.LLM_RESPONSE_MODEL.value, value=response_obj.get("model"), ) - usage = response_obj and response_obj.get("usage") + usage = isinstance(response_obj, dict) and response_obj.get("usage") if usage: self.safe_set_attribute( span=span, @@ -2070,12 +2081,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ############################################# ########## LLM Response Attributes ########## ############################################# - if response_obj is not None: - if response_obj.get("choices"): + if isinstance(response_obj, dict): + choices = response_obj.get("choices") + output_items = response_obj.get("output") + if choices: transformed_choices = ( - self._transform_choices_to_otel_semantic_conventions( - response_obj.get("choices") - ) + self._transform_choices_to_otel_semantic_conventions(choices) ) self.safe_set_attribute( span=span, @@ -2084,7 +2095,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) finish_reasons = [] - for idx, choice in enumerate(response_obj.get("choices")): + for idx, choice in enumerate(choices): if choice.get("finish_reason"): finish_reasons.append(choice.get("finish_reason")) @@ -2095,9 +2106,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=safe_dumps(finish_reasons), ) - for idx, choice in enumerate(response_obj.get("choices")): + for idx, choice in enumerate(choices): if choice.get("finish_reason"): - message = choice.get("message") + message = choice.get("message") or {} tool_calls = message.get("tool_calls") if tool_calls: kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) # type: ignore @@ -2108,12 +2119,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=value, ) - elif response_obj.get("output"): + elif output_items: # Responses API: ResponsesAPIResponse has an "output" # list instead of "choices". Each item with # type="message" contains a "content" list of # OutputText objects (type="output_text"). - output_items = response_obj.get("output") output_messages = self._transform_responses_api_output_to_otel( output_items ) @@ -2333,7 +2343,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # gen_ai.* / metadata.* attributes — duplicating them here doubles # storage and adds noise (Issue #3). litellm_params = kwargs.get("litellm_params", {}) or {} - custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown") + custom_llm_provider = litellm_params.get("custom_llm_provider") or "Unknown" _raw_response = kwargs.get("original_response") _additional_args = kwargs.get("additional_args", {}) or {} diff --git a/tests/test_litellm/integrations/test_opentelemetry_response_obj.py b/tests/test_litellm/integrations/test_opentelemetry_response_obj.py new file mode 100644 index 00000000000..b92e8f64597 --- /dev/null +++ b/tests/test_litellm/integrations/test_opentelemetry_response_obj.py @@ -0,0 +1,204 @@ +""" +Regression tests for OTel callback handling of non-standard response_obj shapes +and non-string message content. + +Covers: +- #24516: response_obj can be a list (Usage AI chat flow) +- #24057: message.content can be list[dict] (multimodal) +- gen_ai.system set to None when custom_llm_provider is explicitly None +""" + +import json +import os +import sys +import unittest +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.abspath("../..")) + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider +from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig + + +class TestOtelNonDictResponseObj(unittest.TestCase): + """Verify _handle_success does not crash when response_obj is a list.""" + + def _make_otel(self, enable_events=False): + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + log_exporter = InMemoryLogExporter() + logger_provider = OTLoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + config = OpenTelemetryConfig(enable_events=enable_events) + otel = OpenTelemetry( + config=config, + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + otel.tracer = tracer_provider.get_tracer(__name__) + return otel, span_exporter + + def _make_kwargs(self, custom_llm_provider="openai"): + return { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": { + "custom_llm_provider": custom_llm_provider, + "proxy_server_request": None, + }, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "hidden_params": {}, + }, + } + + @patch.dict(os.environ, {}, clear=True) + def test_handle_success_with_list_response_obj(self): + """response_obj as a list should not raise (Usage AI chat flow).""" + otel, span_exporter = self._make_otel(enable_events=True) + kwargs = self._make_kwargs() + response_obj = [{"role": "assistant", "content": "Hi there"}] + + start = datetime.utcnow() + end = start + timedelta(seconds=1) + + # Should not raise - covers both _handle_success and _emit_semantic_logs + otel._handle_success(kwargs, response_obj, start, end) + + spans = span_exporter.get_finished_spans() + self.assertTrue(spans, "Expected at least one span even with list response_obj") + + @patch.dict(os.environ, {}, clear=True) + def test_handle_success_with_none_response_obj(self): + """response_obj as None should not raise.""" + otel, span_exporter = self._make_otel(enable_events=True) + kwargs = self._make_kwargs() + + start = datetime.utcnow() + end = start + timedelta(seconds=1) + + otel._handle_success(kwargs, None, start, end) + + spans = span_exporter.get_finished_spans() + self.assertTrue(spans, "Expected at least one span even with None response_obj") + + @patch.dict(os.environ, {}, clear=True) + def test_set_attributes_with_list_response_obj(self): + """set_attributes should not crash when response_obj is a list.""" + otel = OpenTelemetry(config=OpenTelemetryConfig()) + mock_span = MagicMock() + kwargs = self._make_kwargs() + + # Should not raise + otel.set_attributes( + span=mock_span, kwargs=kwargs, response_obj=[{"content": "hi"}] + ) + + @patch.dict(os.environ, {}, clear=True) + def test_set_attributes_with_none_provider(self): + """custom_llm_provider=None should fall back to 'Unknown'.""" + otel = OpenTelemetry(config=OpenTelemetryConfig()) + mock_span = MagicMock() + kwargs = self._make_kwargs(custom_llm_provider=None) + response_obj = { + "id": "test-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # Verify gen_ai.system is exactly "Unknown", not None or empty + found_system = False + for call in mock_span.set_attribute.call_args_list: + args = call[0] if call[0] else () + if len(args) >= 2 and "gen_ai.system" in str(args[0]): + self.assertEqual( + args[1], + "Unknown", + "gen_ai.system should fall back to 'Unknown' when provider is None", + ) + found_system = True + self.assertTrue(found_system, "Expected gen_ai.system attribute to be set") + + +class TestOtelNonStringContent(unittest.TestCase): + """Verify multimodal list[dict] content is serialized, not passed raw.""" + + @patch.dict(os.environ, {}, clear=True) + def test_set_attributes_multimodal_content(self): + """message.content as list[dict] should be serialized to JSON string.""" + otel = OpenTelemetry(config=OpenTelemetryConfig()) + mock_span = MagicMock() + + multimodal_content = [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/img.png"}}, + ] + + kwargs = { + "model": "gpt-4-vision", + "messages": [{"role": "user", "content": multimodal_content}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + response_obj = { + "id": "test-id", + "model": "gpt-4-vision", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "It's a cat.", "role": "assistant"}, + } + ], + "usage": {"prompt_tokens": 50, "completion_tokens": 5, "total_tokens": 55}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # Verify no list was passed as an attribute value, and that content + # attributes contain valid serialized JSON where applicable + for call in mock_span.set_attribute.call_args_list: + args = call[0] if call[0] else () + if len(args) >= 2: + self.assertNotIsInstance( + args[1], + list, + f"Attribute {args[0]} should not be a raw list", + ) + # If it's a content-related attribute with our multimodal data, + # verify it's valid JSON + if "gen_ai.prompt" in str(args[0]) and isinstance(args[1], str): + try: + parsed = json.loads(args[1]) + # Should contain our multimodal content structure + if isinstance(parsed, list) and len(parsed) > 0: + self.assertIn("type", parsed[0]) + except (json.JSONDecodeError, TypeError): + pass # not all content attrs are JSON + + +if __name__ == "__main__": + unittest.main()