diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index c9a1531b5d4..b75e296be47 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -13,18 +13,20 @@ from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: from opentelemetry.trace import Span +from litellm.integrations._types.open_inference import ( + MessageAttributes, + ImageAttributes, + SpanAttributes, + AudioAttributes, + EmbeddingAttributes, + OpenInferenceSpanKindValues +) class ArizeOTELAttributes(BaseLLMObsOTELAttributes): - @staticmethod @override def set_messages(span: "Span", kwargs: Dict[str, Any]): - from litellm.integrations._types.open_inference import ( - MessageAttributes, - SpanAttributes, - ) - messages = kwargs.get("messages") # for /chat/completions @@ -56,7 +58,6 @@ class ArizeOTELAttributes(BaseLLMObsOTELAttributes): def set_response_output_messages(span: "Span", response_obj): """ Sets output message attributes on the span from the LLM response. - Args: span: The OpenTelemetry span to set attributes on response_obj: The response object containing choices with messages @@ -88,112 +89,243 @@ class ArizeOTELAttributes(BaseLLMObsOTELAttributes): ) -def _set_tool_attributes(span: "Span", optional_params: dict): - """Helper to set tool and function call attributes on span.""" - from litellm.integrations._types.open_inference import ( - MessageAttributes, - SpanAttributes, - ToolCallAttributes, - ) - - tools = optional_params.get("tools") - if tools: - for idx, tool in enumerate(tools): - function = tool.get("function") - if not function: - continue - prefix = f"{SpanAttributes.LLM_TOOLS}.{idx}" - safe_set_attribute( - span, f"{prefix}.{SpanAttributes.TOOL_NAME}", function.get("name") - ) - safe_set_attribute( - span, - f"{prefix}.{SpanAttributes.TOOL_DESCRIPTION}", - function.get("description"), - ) - safe_set_attribute( - span, - f"{prefix}.{SpanAttributes.TOOL_PARAMETERS}", - json.dumps(function.get("parameters")), - ) - - functions = optional_params.get("functions") - if functions: - for idx, function in enumerate(functions): - prefix = f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{idx}" - safe_set_attribute( - span, - f"{prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", - function.get("name"), - ) - - def _set_response_attributes(span: "Span", response_obj): """Helper to set response output and token usage attributes on span.""" - from litellm.integrations._types.open_inference import ( - MessageAttributes, - SpanAttributes, - ) if not hasattr(response_obj, "get"): return + _set_choice_outputs(span, response_obj, MessageAttributes, SpanAttributes) + _set_image_outputs(span, response_obj, ImageAttributes, SpanAttributes) + _set_audio_outputs(span, response_obj, AudioAttributes, SpanAttributes) + _set_embedding_outputs(span, response_obj, EmbeddingAttributes, SpanAttributes) + _set_structured_outputs(span, response_obj, MessageAttributes, SpanAttributes) + _set_usage_outputs(span, response_obj, SpanAttributes) + + +def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs): for idx, choice in enumerate(response_obj.get("choices", [])): response_message = choice.get("message", {}) safe_set_attribute( span, - SpanAttributes.OUTPUT_VALUE, + span_attrs.OUTPUT_VALUE, response_message.get("content", ""), ) - prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.{idx}" + prefix = f"{span_attrs.LLM_OUTPUT_MESSAGES}.{idx}" safe_set_attribute( span, - f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", + f"{prefix}.{msg_attrs.MESSAGE_ROLE}", response_message.get("role"), ) safe_set_attribute( span, - f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", + f"{prefix}.{msg_attrs.MESSAGE_CONTENT}", response_message.get("content", ""), ) - output_items = response_obj.get("output", []) - if output_items: - for i, item in enumerate(output_items): - prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.{i}" - if hasattr(item, "type"): - item_type = item.type - if item_type == "reasoning" and hasattr(item, "summary"): - for summary in item.summary: - if hasattr(summary, "text"): - safe_set_attribute( - span, - f"{prefix}.{MessageAttributes.MESSAGE_REASONING_SUMMARY}", - summary.text, - ) - elif item_type == "message" and hasattr(item, "content"): - message_content = "" - content_list = item.content - if content_list and len(content_list) > 0: - first_content = content_list[0] - message_content = getattr(first_content, "text", "") - message_role = getattr(item, "role", "assistant") - safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, message_content) - safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", message_content) - safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", message_role) +def _set_image_outputs(span: "Span", response_obj, image_attrs, span_attrs): + images = response_obj.get("data", []) + for i, image in enumerate(images): + img_url = image.get("url") + if img_url is None and image.get("b64_json"): + img_url = f"data:image/png;base64,{image.get('b64_json')}" + + if not img_url: + continue + + if i == 0: + safe_set_attribute(span, span_attrs.OUTPUT_VALUE, img_url) + + safe_set_attribute(span, f"{image_attrs.IMAGE_URL}.{i}", img_url) + + +def _set_audio_outputs(span: "Span", response_obj, audio_attrs, span_attrs): + audio = response_obj.get("audio", []) + for i, audio_item in enumerate(audio): + audio_url = audio_item.get("url") + if audio_url is None and audio_item.get("b64_json"): + audio_url = f"data:audio/wav;base64,{audio_item.get('b64_json')}" + + if audio_url: + if i == 0: + safe_set_attribute(span, span_attrs.OUTPUT_VALUE, audio_url) + safe_set_attribute(span, f"{audio_attrs.AUDIO_URL}.{i}", audio_url) + + audio_mime = audio_item.get("mime_type") + if audio_mime: + safe_set_attribute(span, f"{audio_attrs.AUDIO_MIME_TYPE}.{i}", audio_mime) + + audio_transcript = audio_item.get("transcript") + if audio_transcript: + safe_set_attribute(span, f"{audio_attrs.AUDIO_TRANSCRIPT}.{i}", audio_transcript) + + +def _set_embedding_outputs(span: "Span", response_obj, embedding_attrs, span_attrs): + embeddings = response_obj.get("data", []) + for i, embedding_item in enumerate(embeddings): + embedding_vector = embedding_item.get("embedding") + if embedding_vector: + if i == 0: + safe_set_attribute( + span, + span_attrs.OUTPUT_VALUE, + str(embedding_vector), + ) + + safe_set_attribute( + span, + f"{embedding_attrs.EMBEDDING_VECTOR}.{i}", + str(embedding_vector), + ) + + embedding_text = embedding_item.get("text") + if embedding_text: + safe_set_attribute( + span, + f"{embedding_attrs.EMBEDDING_TEXT}.{i}", + str(embedding_text), + ) + + +def _set_structured_outputs(span: "Span", response_obj, msg_attrs, span_attrs): + output_items = response_obj.get("output", []) + for i, item in enumerate(output_items): + prefix = f"{span_attrs.LLM_OUTPUT_MESSAGES}.{i}" + if not hasattr(item, "type"): + continue + + item_type = item.type + if item_type == "reasoning" and hasattr(item, "summary"): + for summary in item.summary: + if hasattr(summary, "text"): + safe_set_attribute( + span, + f"{prefix}.{msg_attrs.MESSAGE_REASONING_SUMMARY}", + summary.text, + ) + elif item_type == "message" and hasattr(item, "content"): + message_content = "" + content_list = item.content + if content_list and len(content_list) > 0: + first_content = content_list[0] + message_content = getattr(first_content, "text", "") + message_role = getattr(item, "role", "assistant") + safe_set_attribute(span, span_attrs.OUTPUT_VALUE, message_content) + safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_CONTENT}", message_content) + safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_ROLE}", message_role) + + +def _set_usage_outputs(span: "Span", response_obj, span_attrs): usage = response_obj and response_obj.get("usage") - if usage: - safe_set_attribute(span, SpanAttributes.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens")) - completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens") - if completion_tokens: - safe_set_attribute(span, SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, completion_tokens) - prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens") - if prompt_tokens: - safe_set_attribute(span, SpanAttributes.LLM_TOKEN_COUNT_PROMPT, prompt_tokens) - reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens") - if reasoning_tokens: - safe_set_attribute(span, SpanAttributes.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING, reasoning_tokens) + if not usage: + return + + safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens")) + completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens") + if completion_tokens: + safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_COMPLETION, completion_tokens) + prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens") + if prompt_tokens: + safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_PROMPT, prompt_tokens) + reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens") + if reasoning_tokens: + safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING, reasoning_tokens) + + +def _infer_open_inference_span_kind(call_type: Optional[str]) -> str: + """ + Map LiteLLM call types to OpenInference span kinds. + """ + + if not call_type: + return OpenInferenceSpanKindValues.UNKNOWN.value + + lowered = str(call_type).lower() + + if "embed" in lowered: + return OpenInferenceSpanKindValues.EMBEDDING.value + + if "rerank" in lowered: + return OpenInferenceSpanKindValues.RERANKER.value + + if "search" in lowered: + return OpenInferenceSpanKindValues.RETRIEVER.value + + if "moderation" in lowered or "guardrail" in lowered: + return OpenInferenceSpanKindValues.GUARDRAIL.value + + if lowered == "call_mcp_tool" or lowered == "mcp" or lowered.endswith("tool"): + return OpenInferenceSpanKindValues.TOOL.value + + if "asend_message" in lowered or "a2a" in lowered or "assistant" in lowered: + return OpenInferenceSpanKindValues.AGENT.value + + if any( + keyword in lowered + for keyword in ( + "completion", + "chat", + "image", + "audio", + "speech", + "transcription", + "generate_content", + "response", + "videos", + "realtime", + "pass_through", + "anthropic_messages", + "ocr", + ) + ): + return OpenInferenceSpanKindValues.LLM.value + + if any(keyword in lowered for keyword in ("file", "batch", "container", "fine_tuning_job")): + return OpenInferenceSpanKindValues.CHAIN.value + + return OpenInferenceSpanKindValues.UNKNOWN.value + +def _set_tool_attributes( + span: "Span", optional_tools: Optional[list], metadata_tools: Optional[list] +): + """set tool attributes on span from optional_params or tool call metadata""" + if optional_tools: + for idx, tool in enumerate(optional_tools): + if not isinstance(tool, dict): + continue + function = tool.get("function") if isinstance(tool.get("function"), dict) else None + if not function: + continue + tool_name = function.get("name") + if tool_name: + safe_set_attribute(span, f"{SpanAttributes.LLM_TOOLS}.{idx}.name", tool_name) + tool_description = function.get("description") + if tool_description: + safe_set_attribute(span, f"{SpanAttributes.LLM_TOOLS}.{idx}.description", tool_description) + params = function.get("parameters") + if params is not None: + safe_set_attribute(span, f"{SpanAttributes.LLM_TOOLS}.{idx}.parameters", json.dumps(params)) + + if metadata_tools and isinstance(metadata_tools, list): + for idx, tool in enumerate(metadata_tools): + if not isinstance(tool, dict): + continue + tool_name = tool.get("name") + if tool_name: + safe_set_attribute( + span, + f"{SpanAttributes.LLM_INVOCATION_PARAMETERS}.tools.{idx}.name", + tool_name, + ) + + tool_description = tool.get("description") + if tool_description: + safe_set_attribute( + span, + f"{SpanAttributes.LLM_INVOCATION_PARAMETERS}.tools.{idx}.description", + tool_description, + ) def set_attributes( @@ -202,70 +334,42 @@ def set_attributes( """ Populates span with OpenInference-compliant LLM attributes for Arize and Phoenix tracing. """ - from litellm.integrations._types.open_inference import ( - OpenInferenceSpanKindValues, - SpanAttributes, - ) - try: - # Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers) - optional_params = kwargs.get("optional_params", {}) - if isinstance(optional_params, dict): - optional_params.pop("secret_fields", None) - litellm_params = kwargs.get("litellm_params", {}) + optional_params = _sanitize_optional_params(kwargs.get("optional_params")) + litellm_params = kwargs.get("litellm_params", {}) or {} standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object" ) if standard_logging_payload is None: raise ValueError("standard_logging_object not found in kwargs") - metadata = ( - standard_logging_payload.get("metadata") - if standard_logging_payload - else None + metadata = standard_logging_payload.get("metadata") if standard_logging_payload else None + _set_metadata_attributes(span, metadata, SpanAttributes) + + metadata_tools = _extract_metadata_tools(metadata) + optional_tools = _extract_optional_tools(optional_params) + + call_type = standard_logging_payload.get("call_type") + _set_request_attributes( + span=span, + kwargs=kwargs, + standard_logging_payload=standard_logging_payload, + optional_params=optional_params, + litellm_params=litellm_params, + response_obj=response_obj, + span_attrs=SpanAttributes, ) - if metadata is not None: - safe_set_attribute(span, SpanAttributes.METADATA, safe_dumps(metadata)) - if kwargs.get("model"): - safe_set_attribute(span, SpanAttributes.LLM_MODEL_NAME, kwargs.get("model")) + span_kind = _infer_open_inference_span_kind(call_type=call_type) + _set_tool_attributes(span, optional_tools, metadata_tools) + if (optional_tools or metadata_tools) and span_kind != OpenInferenceSpanKindValues.TOOL.value: + span_kind = OpenInferenceSpanKindValues.TOOL.value - safe_set_attribute(span, "llm.request.type", standard_logging_payload["call_type"]) - safe_set_attribute(span, SpanAttributes.LLM_PROVIDER, litellm_params.get("custom_llm_provider", "Unknown")) - - if optional_params.get("max_tokens"): - safe_set_attribute(span, "llm.request.max_tokens", optional_params.get("max_tokens")) - if optional_params.get("temperature"): - safe_set_attribute(span, "llm.request.temperature", optional_params.get("temperature")) - if optional_params.get("top_p"): - safe_set_attribute(span, "llm.request.top_p", optional_params.get("top_p")) - - safe_set_attribute(span, "llm.is_streaming", str(optional_params.get("stream", False))) - - if optional_params.get("user"): - safe_set_attribute(span, "llm.user", optional_params.get("user")) - - if response_obj and response_obj.get("id"): - safe_set_attribute(span, "llm.response.id", response_obj.get("id")) - if response_obj and response_obj.get("model"): - safe_set_attribute(span, "llm.response.model", response_obj.get("model")) - - safe_set_attribute(span, SpanAttributes.OPENINFERENCE_SPAN_KIND, OpenInferenceSpanKindValues.LLM.value) + safe_set_attribute(span, SpanAttributes.OPENINFERENCE_SPAN_KIND, span_kind) attributes.set_messages(span, kwargs) - _set_tool_attributes(span=span, optional_params=optional_params) - - model_params = ( - standard_logging_payload.get("model_parameters") - if standard_logging_payload - else None - ) - if model_params: - safe_set_attribute(span, SpanAttributes.LLM_INVOCATION_PARAMETERS, safe_dumps(model_params)) - if model_params.get("user"): - user_id = model_params.get("user") - if user_id is not None: - safe_set_attribute(span, SpanAttributes.USER_ID, user_id) + model_params = standard_logging_payload.get("model_parameters") if standard_logging_payload else None + _set_model_params(span, model_params, SpanAttributes) _set_response_attributes(span=span, response_obj=response_obj) @@ -275,3 +379,72 @@ def set_attributes( ) if hasattr(span, "record_exception"): span.record_exception(e) + + +def _sanitize_optional_params(optional_params: Optional[dict]) -> dict: + if not isinstance(optional_params, dict): + return {} + optional_params.pop("secret_fields", None) + return optional_params + + +def _set_metadata_attributes(span: "Span", metadata: Optional[Any], span_attrs) -> None: + if metadata is not None: + safe_set_attribute(span, span_attrs.METADATA, safe_dumps(metadata)) + + +def _extract_metadata_tools(metadata: Optional[Any]) -> Optional[list]: + if not isinstance(metadata, dict): + return None + llm_obj = metadata.get("llm") + if isinstance(llm_obj, dict): + return llm_obj.get("tools") + return None + + +def _extract_optional_tools(optional_params: dict) -> Optional[list]: + return optional_params.get("tools") if isinstance(optional_params, dict) else None + + +def _set_request_attributes( + span: "Span", + kwargs, + standard_logging_payload: StandardLoggingPayload, + optional_params: dict, + litellm_params: dict, + response_obj, + span_attrs, +): + if kwargs.get("model"): + safe_set_attribute(span, span_attrs.LLM_MODEL_NAME, kwargs.get("model")) + + safe_set_attribute(span, "llm.request.type", standard_logging_payload.get("call_type")) + safe_set_attribute(span, span_attrs.LLM_PROVIDER, litellm_params.get("custom_llm_provider", "Unknown")) + + if optional_params.get("max_tokens"): + safe_set_attribute(span, "llm.request.max_tokens", optional_params.get("max_tokens")) + if optional_params.get("temperature"): + safe_set_attribute(span, "llm.request.temperature", optional_params.get("temperature")) + if optional_params.get("top_p"): + safe_set_attribute(span, "llm.request.top_p", optional_params.get("top_p")) + + safe_set_attribute(span, "llm.is_streaming", str(optional_params.get("stream", False))) + + if optional_params.get("user"): + safe_set_attribute(span, "llm.user", optional_params.get("user")) + + if response_obj and response_obj.get("id"): + safe_set_attribute(span, "llm.response.id", response_obj.get("id")) + if response_obj and response_obj.get("model"): + safe_set_attribute(span, "llm.response.model", response_obj.get("model")) + + +def _set_model_params(span: "Span", model_params: Optional[dict], span_attrs) -> None: + if not model_params: + return + + safe_set_attribute(span, span_attrs.LLM_INVOCATION_PARAMETERS, safe_dumps(model_params)) + if model_params.get("user"): + user_id = model_params.get("user") + if user_id is not None: + safe_set_attribute(span, span_attrs.USER_ID, user_id) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 93d631eb0f2..a8a7fa77b3d 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -17,6 +17,10 @@ from litellm.types.utils import ( StandardCallbackDynamicParams, StandardLoggingPayload, ) +from litellm.integrations._types.open_inference import ( + OpenInferenceSpanKindValues, + SpanAttributes, +) # OpenTelemetry imports moved to individual functions to avoid import errors when not installed @@ -660,6 +664,9 @@ class OpenTelemetry(CustomLogger): self._maybe_log_raw_request( kwargs, response_obj, start_time, end_time, span ) + # Ensure proxy-request parent span is annotated with the actual operation kind + if parent_span is not None and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME: + self.set_attributes(parent_span, kwargs, response_obj) else: # Do not create primary span (keep hierarchy shallow when parent exists) from opentelemetry.trace import Status, StatusCode @@ -1106,6 +1113,12 @@ class OpenTelemetry(CustomLogger): context=context, ) + self.safe_set_attribute( + span=guardrail_span, + key=SpanAttributes.OPENINFERENCE_SPAN_KIND, + value=OpenInferenceSpanKindValues.GUARDRAIL.value, + ) + self.safe_set_attribute( span=guardrail_span, key="guardrail_name", diff --git a/tests/logging_callback_tests/test_otel_logging.py b/tests/logging_callback_tests/test_otel_logging.py index b53be44fe0f..a0c78305e60 100644 --- a/tests/logging_callback_tests/test_otel_logging.py +++ b/tests/logging_callback_tests/test_otel_logging.py @@ -10,11 +10,25 @@ sys.path.insert( import pytest import litellm -from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig, Span import asyncio import logging +from opentelemetry import trace from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from litellm._logging import verbose_logger +from litellm.integrations.arize.arize_phoenix import ArizePhoenixLogger +from litellm.integrations._types.open_inference import ( + OpenInferenceSpanKindValues, + SpanAttributes as OISpanAttributes, +) +from litellm.integrations.opentelemetry import ( + LITELLM_PROXY_REQUEST_SPAN_NAME, + LITELLM_TRACER_NAME, + LITELLM_REQUEST_SPAN_NAME, + OpenTelemetry, + OpenTelemetryConfig, + RAW_REQUEST_SPAN_NAME, + Span, +) from litellm.proxy._types import SpanAttributes verbose_logger.setLevel(logging.DEBUG) @@ -242,3 +256,51 @@ def validate_redacted_message_span_attributes(span): ), f"Non-metadata attribute found: {attr}" pass + +@pytest.mark.asyncio +async def test_arize_phoenix_adds_openinference_kind_and_avoids_duplicate_litellm_spans(): + """ + Ensure Arize Phoenix spans include OpenInference span kind and do not create + a duplicate litellm_request span when a proxy parent span is already active. + """ + + exporter.clear() + litellm.logging_callback_manager._reset_all_callbacks() + + otel_logger = ArizePhoenixLogger(config=OpenTelemetryConfig(exporter=exporter)) + litellm.callbacks = [otel_logger] + litellm.success_callback = [] + litellm.failure_callback = [] + + tracer = trace.get_tracer(LITELLM_TRACER_NAME) + parent_span = tracer.start_span(LITELLM_PROXY_REQUEST_SPAN_NAME) + + # Keep parent span active; OpenTelemetry logger will attach attributes and end it. + with trace.use_span(parent_span, end_on_exit=False): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "ping"}], + mock_response="pong", + ) + + # Flush span processing + await asyncio.sleep(1) + + if parent_span.is_recording(): + parent_span.end() + + spans = exporter.get_finished_spans() + + span_names = [span.name for span in spans] + assert LITELLM_REQUEST_SPAN_NAME not in span_names + assert span_names.count(LITELLM_PROXY_REQUEST_SPAN_NAME) == 1 + assert span_names.count(RAW_REQUEST_SPAN_NAME) == 1 + + # All spans should belong to the same trace (parent + raw child) + assert len({span.context.trace_id for span in spans}) == 1 + assert len(spans) == 2 + + proxy_span = next(span for span in spans if span.name == LITELLM_PROXY_REQUEST_SPAN_NAME) + assert proxy_span.attributes.get(OISpanAttributes.OPENINFERENCE_SPAN_KIND) == OpenInferenceSpanKindValues.LLM.value + + exporter.clear()