scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 99

fix(otel): guard response_obj type and serialize non-string content

Browse source 
* Replace truthy response_obj checks with isinstance(response_obj, dict)
  in _record_metrics, _record_tpot_metric, _emit_semantic_logs, and the
  span attribute setters so non-dict payloads (None, Pydantic models,
  raw strings from embeddings/images) no longer raise AttributeError.

* Serialize non-string message content via safe_dumps before assigning
  to gen_ai.prompt and message.content event attributes, so multimodal
  payloads (lists of parts, dicts) no longer trip OTEL's str-only
  attribute validation.

* Make custom_llm_provider fallback null-safe in _record_metrics,
  _emit_semantic_logs, set_attributes, and set_raw_request_attributes
  by switching from dict.get(key, "Unknown") to dict.get(key) or
  "Unknown" (the two-arg form returns None when the key is explicitly
  set to None, which happens for some embedding paths).

* Guard choice.get("message") with or {} so a choice with a missing
  message dict no longer raises in the tool_calls extraction loop.

Tests cover non-dict response_obj, multimodal content, and the None
provider fallback for both event and span code paths.
This commit is contained in:
Nik-Reddy 2026-05-17 20:26:51 -07:00
parent cf9b5e4fa7
commit b3fa24d5f5
2 changed files with 236 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
litellm/integrations/opentelemetry.py
View file

@ -1075,7 +1075,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
def _record_metrics(self, kwargs, response_obj, start_time, end_time):
duration_s = (end_time - start_time).total_seconds()
params = kwargs.get("litellm_params") or {}
provider = params.get("custom_llm_provider", "Unknown")
provider = params.get("custom_llm_provider") or "Unknown"
common_attrs = {
"gen_ai.operation.name": (
@ -1122,7 +1122,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
duration_s, attributes=common_attrs
)
if (
response_obj
isinstance(response_obj, dict)
and (usage := response_obj.get("usage"))
and self._token_usage_histogram
):
@ -1211,7 +1211,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# Get completion tokens from response_obj
completion_tokens = None
if response_obj and (usage := response_obj.get("usage")):
if isinstance(response_obj, dict) and (usage := response_obj.get("usage")):
completion_tokens = usage.get("completion_tokens")
if completion_tokens is None or completion_tokens <= 0:
@ -1327,6 +1327,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if not self.config.enable_events:
return
if not isinstance(response_obj, dict):
return
# NOTE: Semantic logs (gen_ai.content.prompt/completion events) have compatibility issues
# with OTEL SDK >= 1.39.0 due to breaking changes in PR #4676:
# - LogRecord moved from opentelemetry.sdk._logs to opentelemetry.sdk._logs._internal
@ -1345,8 +1348,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
parent_ctx = span.get_span_context()
provider = (kwargs.get("litellm_params") or {}).get(
"custom_llm_provider", "Unknown"
)
"custom_llm_provider"
) or "Unknown"
if self._gen_ai_semconv_latest_experimental:
self._emit_inference_details_event(
@ -1369,7 +1372,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
attrs["id"] = msg["id"]
capture_event_content = self._capture_in_event()
if capture_event_content and msg.get("content"):
attrs["gen_ai.prompt"] = msg["content"]
content = msg["content"]
if isinstance(content, str):
attrs["gen_ai.prompt"] = content
else:
attrs["gen_ai.prompt"] = safe_dumps(content)
body = msg.copy()
if not capture_event_content:
@ -1398,7 +1405,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
body_msg = choice.get("message", {})
capture_event_content = self._capture_in_event()
if capture_event_content and body_msg.get("content"):
attrs["message.content"] = body_msg["content"]
completion_content = body_msg["content"]
if isinstance(completion_content, str):
attrs["message.content"] = completion_content
else:
attrs["message.content"] = safe_dumps(completion_content)
body = {
"index": idx,
"finish_reason": choice.get("finish_reason"),
@ -1876,7 +1887,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
# The Generative AI Provider: Azure, OpenAI, etc.
provider_name = litellm_params.get("custom_llm_provider", "Unknown")
provider_name = litellm_params.get("custom_llm_provider") or "Unknown"
# Latest-experimental semconv replaced gen_ai.system with
# gen_ai.provider.name; emit only the conformant key in that mode.
if self._gen_ai_semconv_latest_experimental:
@ -1941,7 +1952,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# the litellm call ID so every call type can be correlated
# across LiteLLM UI, Phoenix traces, and provider logs (Issue #8).
response_id = (
response_obj.get("id") if response_obj else None
response_obj.get("id") if isinstance(response_obj, dict) else None
) or standard_logging_payload.get("id")
if response_id:
self.safe_set_attribute(
@ -1959,14 +1970,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
# The model used to generate the response.
if response_obj and response_obj.get("model"):
if isinstance(response_obj, dict) and response_obj.get("model"):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_RESPONSE_MODEL.value,
value=response_obj.get("model"),
)
usage = response_obj and response_obj.get("usage")
usage = isinstance(response_obj, dict) and response_obj.get("usage")
if usage:
self.safe_set_attribute(
span=span,
@ -2070,12 +2081,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
#############################################
########## LLM Response Attributes ##########
#############################################
if response_obj is not None:
if response_obj.get("choices"):
if isinstance(response_obj, dict):
choices = response_obj.get("choices")
output_items = response_obj.get("output")
if choices:
transformed_choices = (
self._transform_choices_to_otel_semantic_conventions(
response_obj.get("choices")
)
self._transform_choices_to_otel_semantic_conventions(choices)
)
self.safe_set_attribute(
span=span,
@ -2084,7 +2095,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
finish_reasons = []
for idx, choice in enumerate(response_obj.get("choices")):
for idx, choice in enumerate(choices):
if choice.get("finish_reason"):
finish_reasons.append(choice.get("finish_reason"))
@ -2095,9 +2106,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=safe_dumps(finish_reasons),
)
for idx, choice in enumerate(response_obj.get("choices")):
for idx, choice in enumerate(choices):
if choice.get("finish_reason"):
message = choice.get("message")
message = choice.get("message") or {}
tool_calls = message.get("tool_calls")
if tool_calls:
kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) # type: ignore
@ -2108,12 +2119,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=value,
)
elif response_obj.get("output"):
elif output_items:
# Responses API: ResponsesAPIResponse has an "output"
# list instead of "choices". Each item with
# type="message" contains a "content" list of
# OutputText objects (type="output_text").
output_items = response_obj.get("output")
output_messages = self._transform_responses_api_output_to_otel(
output_items
)
@ -2333,7 +2343,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# gen_ai.* / metadata.* attributes — duplicating them here doubles
# storage and adds noise (Issue #3).
litellm_params = kwargs.get("litellm_params", {}) or {}
custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown")
custom_llm_provider = litellm_params.get("custom_llm_provider") or "Unknown"
_raw_response = kwargs.get("original_response")
_additional_args = kwargs.get("additional_args", {}) or {}

204
tests/test_litellm/integrations/test_opentelemetry_response_obj.py Normal file
View file

@ -0,0 +1,204 @@
"""
Regression tests for OTel callback handling of non-standard response_obj shapes
and non-string message content.
Covers:
- #24516: response_obj can be a list (Usage AI chat flow)
- #24057: message.content can be list[dict] (multimodal)
- gen_ai.system set to None when custom_llm_provider is explicitly None
"""
import json
import os
import sys
import unittest
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
sys.path.insert(0, os.path.abspath("../.."))
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
class TestOtelNonDictResponseObj(unittest.TestCase):
"""Verify _handle_success does not crash when response_obj is a list."""
def _make_otel(self, enable_events=False):
span_exporter = InMemorySpanExporter()
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
metric_reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[metric_reader])
log_exporter = InMemoryLogExporter()
logger_provider = OTLoggerProvider()
logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
config = OpenTelemetryConfig(enable_events=enable_events)
otel = OpenTelemetry(
config=config,
tracer_provider=tracer_provider,
meter_provider=meter_provider,
logger_provider=logger_provider,
)
otel.tracer = tracer_provider.get_tracer(__name__)
return otel, span_exporter
def _make_kwargs(self, custom_llm_provider="openai"):
return {
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"optional_params": {},
"litellm_params": {
"custom_llm_provider": custom_llm_provider,
"proxy_server_request": None,
},
"standard_logging_object": {
"id": "test-id",
"call_type": "completion",
"metadata": {},
"hidden_params": {},
},
}
@patch.dict(os.environ, {}, clear=True)
def test_handle_success_with_list_response_obj(self):
"""response_obj as a list should not raise (Usage AI chat flow)."""
otel, span_exporter = self._make_otel(enable_events=True)
kwargs = self._make_kwargs()
response_obj = [{"role": "assistant", "content": "Hi there"}]
start = datetime.utcnow()
end = start + timedelta(seconds=1)
# Should not raise - covers both _handle_success and _emit_semantic_logs
otel._handle_success(kwargs, response_obj, start, end)
spans = span_exporter.get_finished_spans()
self.assertTrue(spans, "Expected at least one span even with list response_obj")
@patch.dict(os.environ, {}, clear=True)
def test_handle_success_with_none_response_obj(self):
"""response_obj as None should not raise."""
otel, span_exporter = self._make_otel(enable_events=True)
kwargs = self._make_kwargs()
start = datetime.utcnow()
end = start + timedelta(seconds=1)
otel._handle_success(kwargs, None, start, end)
spans = span_exporter.get_finished_spans()
self.assertTrue(spans, "Expected at least one span even with None response_obj")
@patch.dict(os.environ, {}, clear=True)
def test_set_attributes_with_list_response_obj(self):
"""set_attributes should not crash when response_obj is a list."""
otel = OpenTelemetry(config=OpenTelemetryConfig())
mock_span = MagicMock()
kwargs = self._make_kwargs()
# Should not raise
otel.set_attributes(
span=mock_span, kwargs=kwargs, response_obj=[{"content": "hi"}]
)
@patch.dict(os.environ, {}, clear=True)
def test_set_attributes_with_none_provider(self):
"""custom_llm_provider=None should fall back to 'Unknown'."""
otel = OpenTelemetry(config=OpenTelemetryConfig())
mock_span = MagicMock()
kwargs = self._make_kwargs(custom_llm_provider=None)
response_obj = {
"id": "test-id",
"model": "gpt-4",
"choices": [],
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
}
otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj)
# Verify gen_ai.system is exactly "Unknown", not None or empty
found_system = False
for call in mock_span.set_attribute.call_args_list:
args = call[0] if call[0] else ()
if len(args) >= 2 and "gen_ai.system" in str(args[0]):
self.assertEqual(
args[1],
"Unknown",
"gen_ai.system should fall back to 'Unknown' when provider is None",
)
found_system = True
self.assertTrue(found_system, "Expected gen_ai.system attribute to be set")
class TestOtelNonStringContent(unittest.TestCase):
"""Verify multimodal list[dict] content is serialized, not passed raw."""
@patch.dict(os.environ, {}, clear=True)
def test_set_attributes_multimodal_content(self):
"""message.content as list[dict] should be serialized to JSON string."""
otel = OpenTelemetry(config=OpenTelemetryConfig())
mock_span = MagicMock()
multimodal_content = [
{"type": "text", "text": "What is in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/img.png"}},
]
kwargs = {
"model": "gpt-4-vision",
"messages": [{"role": "user", "content": multimodal_content}],
"optional_params": {},
"litellm_params": {"custom_llm_provider": "openai"},
"standard_logging_object": {
"id": "test-id",
"call_type": "completion",
"metadata": {},
},
}
response_obj = {
"id": "test-id",
"model": "gpt-4-vision",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {"content": "It's a cat.", "role": "assistant"},
}
],
"usage": {"prompt_tokens": 50, "completion_tokens": 5, "total_tokens": 55},
}
otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj)
# Verify no list was passed as an attribute value, and that content
# attributes contain valid serialized JSON where applicable
for call in mock_span.set_attribute.call_args_list:
args = call[0] if call[0] else ()
if len(args) >= 2:
self.assertNotIsInstance(
args[1],
list,
f"Attribute {args[0]} should not be a raw list",
)
# If it's a content-related attribute with our multimodal data,
# verify it's valid JSON
if "gen_ai.prompt" in str(args[0]) and isinstance(args[1], str):
try:
parsed = json.loads(args[1])
# Should contain our multimodal content structure
if isinstance(parsed, list) and len(parsed) > 0:
self.assertIn("type", parsed[0])
except (json.JSONDecodeError, TypeError):
pass # not all content attrs are JSON
if __name__ == "__main__":
unittest.main()