scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

fix(opentelemetry): guard against non-dict response_obj in OTEL callbacks

Browse source 
When response_obj is a list (e.g. from Usage AI chat), calling .get()
on it raises AttributeError. Add isinstance(response_obj, dict) checks
before all .get() calls on response_obj in set_attributes,
_record_metrics, _record_time_per_output_token_metric, and
_emit_semantic_logs.

Also fix gen_ai.system attribute being set to None when
custom_llm_provider is explicitly None in litellm_params, which causes
OpenTelemetry SDK to reject the attribute value. Use `or "Unknown"`
instead of default parameter to handle both missing and None cases.

Fixes #24516
This commit is contained in:
Krishna Chaitanya Balusu 2026-03-24 22:13:34 -04:00
parent d93ee444fd
commit 8bd7359057
2 changed files with 146 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
litellm/integrations/opentelemetry.py
View file

@ -655,9 +655,9 @@ class OpenTelemetry(CustomLogger):
def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
"""Extract dynamic headers from kwargs if available."""
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params")
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params")
)
if not standard_callback_dynamic_params:
return None
@ -834,7 +834,7 @@ class OpenTelemetry(CustomLogger):
def _record_metrics(self, kwargs, response_obj, start_time, end_time):
duration_s = (end_time - start_time).total_seconds()
params = kwargs.get("litellm_params") or {}
provider = params.get("custom_llm_provider", "Unknown")
provider = params.get("custom_llm_provider") or "Unknown"
common_attrs = {
"gen_ai.operation.name": "chat",
@ -878,6 +878,7 @@ class OpenTelemetry(CustomLogger):
)
if (
response_obj
and isinstance(response_obj, dict)
and (usage := response_obj.get("usage"))
and self._token_usage_histogram
):
@ -966,7 +967,11 @@ class OpenTelemetry(CustomLogger):
# Get completion tokens from response_obj
completion_tokens = None
if response_obj and (usage := response_obj.get("usage")):
if (
response_obj
and isinstance(response_obj, dict)
and (usage := response_obj.get("usage"))
):
completion_tokens = usage.get("completion_tokens")
if completion_tokens is None or completion_tokens <= 0:
@ -1088,8 +1093,8 @@ class OpenTelemetry(CustomLogger):
parent_ctx = span.get_span_context()
provider = (kwargs.get("litellm_params") or {}).get(
"custom_llm_provider", "Unknown"
)
"custom_llm_provider"
) or "Unknown"
# per-message events
for msg in kwargs.get("messages", []):
@ -1116,7 +1121,10 @@ class OpenTelemetry(CustomLogger):
otel_logger.emit(log_record)
# per-choice events
for idx, choice in enumerate(response_obj.get("choices", [])):
choices = (
response_obj.get("choices", []) if isinstance(response_obj, dict) else []
)
for idx, choice in enumerate(choices):
attrs = {
"event_name": "gen_ai.content.completion",
"gen_ai.system": provider,
@ -1560,7 +1568,7 @@ class OpenTelemetry(CustomLogger):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_SYSTEM.value,
value=litellm_params.get("custom_llm_provider", "Unknown"),
value=litellm_params.get("custom_llm_provider") or "Unknown",
)
# The maximum number of tokens the LLM generates for a request.
@ -1606,7 +1614,9 @@ class OpenTelemetry(CustomLogger):
# the litellm call ID so every call type can be correlated
# across LiteLLM UI, Phoenix traces, and provider logs (Issue #8).
response_id = (
response_obj.get("id") if response_obj else None
response_obj.get("id")
if response_obj and isinstance(response_obj, dict)
else None
) or standard_logging_payload.get("id")
if response_id:
self.safe_set_attribute(
@ -1616,14 +1626,22 @@ class OpenTelemetry(CustomLogger):
)
# The model used to generate the response.
if response_obj and response_obj.get("model"):
if (
response_obj
and isinstance(response_obj, dict)
and response_obj.get("model")
):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_RESPONSE_MODEL.value,
value=response_obj.get("model"),
)
usage = response_obj and response_obj.get("usage")
usage = (
response_obj
and isinstance(response_obj, dict)
and response_obj.get("usage")
)
if usage:
self.safe_set_attribute(
span=span,
@ -1701,7 +1719,7 @@ class OpenTelemetry(CustomLogger):
#############################################
########## LLM Response Attributes ##########
#############################################
if response_obj is not None:
if response_obj is not None and isinstance(response_obj, dict):
if response_obj.get("choices"):
transformed_choices = (
self._transform_choices_to_otel_semantic_conventions(

115
tests/test_litellm/integrations/test_opentelemetry.py
View file

@ -2751,3 +2751,118 @@ class TestResponseIdFallback(unittest.TestCase):
mock_span.set_attribute.assert_any_call(
"gen_ai.response.id", "litellm-img-call-101"
)
class TestOpenTelemetryNonDictResponseObj(unittest.TestCase):
"""Issue #24516: OpenTelemetry callback crashes when response_obj is a
non-dict type (e.g. a list) as can happen with Usage AI chat."""
def _make_kwargs(self, provider="openai"):
return {
"model": "gpt-4",
"optional_params": {},
"litellm_params": {"custom_llm_provider": provider},
"standard_logging_object": {
"id": "litellm-call-id-abc",
"call_type": "completion",
"metadata": {},
},
}
def test_set_attributes_with_list_response_obj(self):
"""set_attributes should not crash when response_obj is a list."""
otel = OpenTelemetry()
mock_span = MagicMock()
kwargs = self._make_kwargs()
response_obj = [{"some": "data"}] # non-dict response
# Should not raise
otel.set_attributes(mock_span, kwargs, response_obj)
# Should still set the fallback response id from standard_logging_payload
mock_span.set_attribute.assert_any_call(
"gen_ai.response.id", "litellm-call-id-abc"
)
def test_set_attributes_with_none_response_obj(self):
"""set_attributes should handle None response_obj gracefully."""
otel = OpenTelemetry()
mock_span = MagicMock()
kwargs = self._make_kwargs()
# Should not raise
otel.set_attributes(mock_span, kwargs, None)
# Should still set the fallback response id
mock_span.set_attribute.assert_any_call(
"gen_ai.response.id", "litellm-call-id-abc"
)
def test_set_attributes_with_none_provider(self):
"""set_attributes should default gen_ai.system to 'Unknown' when
custom_llm_provider is None."""
otel = OpenTelemetry()
mock_span = MagicMock()
kwargs = self._make_kwargs(provider=None)
response_obj = {"id": "resp-1", "choices": [], "usage": None}
otel.set_attributes(mock_span, kwargs, response_obj)
# gen_ai.system should be "Unknown", not None
from litellm.proxy._types import SpanAttributes
mock_span.set_attribute.assert_any_call(
SpanAttributes.LLM_SYSTEM.value, "Unknown"
)
def test_record_metrics_with_list_response_obj(self):
"""_record_metrics should not crash when response_obj is a list."""
otel = OpenTelemetry()
otel._operation_duration_histogram = MagicMock()
otel._token_usage_histogram = MagicMock()
otel._cost_histogram = None
start = datetime.now()
end = start + timedelta(seconds=1)
kwargs = {
"model": "gpt-4",
"litellm_params": {"custom_llm_provider": "openai"},
"standard_logging_object": {"metadata": {}},
}
response_obj = ["item1", "item2"]
# Should not raise
otel._record_metrics(kwargs, response_obj, start, end)
# Duration histogram should still be recorded
otel._operation_duration_histogram.record.assert_called_once()
# Token histogram should NOT be recorded (response_obj is not dict)
otel._token_usage_histogram.record.assert_not_called()
def test_record_metrics_with_none_provider(self):
"""_record_metrics should use 'Unknown' when provider is None."""
otel = OpenTelemetry()
otel._operation_duration_histogram = MagicMock()
otel._token_usage_histogram = None
otel._cost_histogram = None
start = datetime.now()
end = start + timedelta(seconds=1)
kwargs = {
"model": "gpt-4",
"litellm_params": {"custom_llm_provider": None},
"standard_logging_object": {"metadata": {}},
}
response_obj = {
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}
}
otel._record_metrics(kwargs, response_obj, start, end)
call_attrs = otel._operation_duration_histogram.record.call_args
assert call_attrs[1]["attributes"]["gen_ai.system"] == "Unknown"