From c0d506b55733251ec952d4d7064120bbd9871cdc Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 4 Nov 2026 17:09:49 +0000 Subject: [PATCH] feat(otel): support GenAI client suppression and context attribute propagation --- litellm/integrations/otel/emitter.py | 33 +- litellm/integrations/otel/genai_context.py | 127 ++++++ litellm/integrations/otel/logger.py | 96 ++++- litellm/integrations/otel/plumbing/metrics.py | 5 + pyproject.toml | 3 + .../test_otel_genai_context_suppression.py | 407 ++++++++++++++++++ 6 files changed, 645 insertions(+), 26 deletions(-) create mode 100644 litellm/integrations/otel/genai_context.py create mode 100644 tests/unit/integrations/otel/test_otel_genai_context_suppression.py diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 87a432ee7ce..d1ed47dae12 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -228,6 +228,25 @@ class SpanEmitter: # -- the engine ---------------------------------------------------------- # + def _map_data(self, data: SpanData) -> tuple[dict[str, AttrValue], Mapping[str, AttrValue]]: + configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()} + for_langfuse: Final = ( + dict( + sorted( + self._langfuse.map(data).items(), + key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES, + ) + ) + if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends() + else _NO_ATTRIBUTES + ) + return {**configured, **for_langfuse}, for_langfuse + + def map_attributes(self, data: SpanData) -> dict[str, AttrValue]: + """Run the configured mapper chain on ``data`` without creating a span.""" + mapped, _ = self._map_data(data) + return mapped + def emit( self, role: SpanRole, @@ -299,18 +318,8 @@ class SpanEmitter: ) else None ) - configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()} - for_langfuse: Final = ( - dict( - sorted( - self._langfuse.map(data).items(), - key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES, - ) - ) - if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends() - else _NO_ATTRIBUTES - ) - mapped: Final = MappingProxyType({**configured, **for_langfuse}) + raw_mapped, for_langfuse = self._map_data(data) + mapped: Final = MappingProxyType(raw_mapped) stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES reserved: Final = len(stamped_later.keys() - mapped.keys()) overwrites: Final = carried_keys(span) & mapped.keys() diff --git a/litellm/integrations/otel/genai_context.py b/litellm/integrations/otel/genai_context.py new file mode 100644 index 00000000000..9a2f2f3521b --- /dev/null +++ b/litellm/integrations/otel/genai_context.py @@ -0,0 +1,127 @@ +"""OpenTelemetry GenAI client context suppression & attribute propagation. + +Integrates ``OpenTelemetryV2`` with ``opentelemetry-util-genai``'s context-sharing +contract (see ``open-telemetry/semantic-conventions-genai#682`` and +``open-telemetry/opentelemetry-python-genai#859``): + +When LiteLLM is called inside an outer GenAI instrumentation that attached +``InferenceData``, ``EmbeddingData``, ``RetrievalData``, or +``FetchResponseData`` under ``CLIENT__CONTEXT_KEY`` in +``opentelemetry.context``, LiteLLM: +- Suppresses its duplicate client span, ``gen_ai.client.*`` metrics, and + operation-exception log events. +- Publishes its mapped span attributes and ``metric_attributes`` into the + active context data container. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, cast + +from opentelemetry.context import Context, get_value +from opentelemetry.util.genai._invocation import _ContextData +from opentelemetry.util.genai.invocation import ( + CLIENT_EMBEDDING_CONTEXT_KEY, + CLIENT_FETCH_RESPONSE_CONTEXT_KEY, + CLIENT_INFERENCE_CONTEXT_KEY, + CLIENT_RETRIEVAL_CONTEXT_KEY, + EmbeddingData, + FetchResponseData, + InferenceData, + RetrievalData, +) +from opentelemetry.util.types import AttributeValue + +from litellm.integrations.otel.model.payloads import LLMCallSpanData +from litellm.integrations.otel.model.semconv import Error, GenAIOperation + +__all__ = [ + "CLIENT_EMBEDDING_CONTEXT_KEY", + "CLIENT_FETCH_RESPONSE_CONTEXT_KEY", + "CLIENT_INFERENCE_CONTEXT_KEY", + "CLIENT_RETRIEVAL_CONTEXT_KEY", + "EmbeddingData", + "FetchResponseData", + "InferenceData", + "RetrievalData", + "context_spec_for_operation", + "get_active_genai_context_data", + "publish_to_genai_context", +] + +_INFERENCE_OPERATIONS: Final[frozenset[str]] = frozenset( + { + GenAIOperation.CHAT.value, + GenAIOperation.TEXT_COMPLETION.value, + GenAIOperation.GENERATE_CONTENT.value, + } +) + +_FETCH_RESPONSE_CALL_TYPES: Final[frozenset[str]] = frozenset( + { + "get_responses", + "aget_responses", + } +) + + +def context_spec_for_operation( + operation: GenAIOperation | str | None, + call_type: str | None = None, +) -> tuple[str, type[_ContextData]] | None: + """Return ``(context_key, dataclass_type)`` for a GenAI client operation, or ``None``.""" + op_str: Final = operation.value if isinstance(operation, GenAIOperation) else (operation or "") + if call_type and call_type.lower() in _FETCH_RESPONSE_CALL_TYPES: + return CLIENT_FETCH_RESPONSE_CONTEXT_KEY, FetchResponseData + if op_str == "fetch_response": + return CLIENT_FETCH_RESPONSE_CONTEXT_KEY, FetchResponseData + if op_str in _INFERENCE_OPERATIONS: + return CLIENT_INFERENCE_CONTEXT_KEY, InferenceData + if op_str == GenAIOperation.EMBEDDINGS.value: + return CLIENT_EMBEDDING_CONTEXT_KEY, EmbeddingData + if op_str == GenAIOperation.RETRIEVAL.value: + return CLIENT_RETRIEVAL_CONTEXT_KEY, RetrievalData + return None + + +def get_active_genai_context_data( + operation: GenAIOperation | str | None, + *, + call_type: str | None = None, + context: Context | None = None, +) -> _ContextData | None: + """Return the active outer ``Data`` from ``opentelemetry.context`` if present.""" + spec: Final = context_spec_for_operation(operation, call_type=call_type) + if spec is None: + return None + context_key, data_cls = spec + for candidate_ctx in (context, None): + value = get_value(context_key, context=candidate_ctx) + if isinstance(value, data_cls): + return value + if candidate_ctx is None: + break + return None + + +def publish_to_genai_context( + ctx_data: _ContextData, + data: LLMCallSpanData, + span_attrs: Mapping[str, object], + metric_attrs: Mapping[str, object] | None = None, +) -> None: + """Publish LiteLLM's mapped span and metric attributes into an outer ``ctx_data``.""" + for key, val in span_attrs.items(): + if val is not None and isinstance(val, (str, bool, int, float, list, tuple)): + ctx_data.attributes[key] = cast(AttributeValue, val) + + if data.error is not None: + ctx_data.attributes[Error.TYPE] = data.error.error_type + if data.error.message: + ctx_data.attributes[Error.MESSAGE] = data.error.message + + if metric_attrs: + for key, val in metric_attrs.items(): + if val is not None and isinstance(val, (str, bool, int, float)): + ctx_data.metric_attributes[key] = val diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index b5b65d299e0..3370478ec56 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -29,6 +29,10 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.emitter import SpanEmitter, stamp_error +from litellm.integrations.otel.genai_context import ( + get_active_genai_context_data, + publish_to_genai_context, +) from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.model.baggage import promoted_baggage from litellm.integrations.otel.model.config import OpenTelemetryV2Config, excluded_db_systems_from @@ -51,7 +55,7 @@ from litellm.integrations.otel.model.payloads import ( ) from litellm.integrations.otel.model.semconv import Error from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service -from litellm.integrations.otel.model.utils import to_ns +from litellm.integrations.otel.model.utils import as_str, to_ns from litellm.integrations.otel.plumbing.context import ( active_phase, active_phase_span, @@ -169,17 +173,19 @@ class _LLMCallSpan: under it, and must be released exactly once when the carrier is removed. """ - __slots__ = ("provider", "span", "start_time_ns") + __slots__ = ("outer_ctx_data", "provider", "span", "start_time_ns") def __init__( self, span: "Span | None", start_time_ns: int | None, provider: "TracerProvider | None" = None, + outer_ctx_data: "object | None" = None, ) -> None: self.span = span self.start_time_ns = start_time_ns self.provider = provider + self.outer_ctx_data = outer_ctx_data def _llm_call_parent_context(call: LLMCallEvent) -> Context: @@ -332,6 +338,17 @@ class OpenTelemetryV2(CustomLogger): # Do not route on the deferred path: creating or LRU-touching a tenant # provider here would evict idle ones even though close re-routes. parent_context: Final = _llm_call_parent_context(call) + outer_ctx_data: Final = get_active_genai_context_data( + call.operation, + call_type=as_str(kwargs.get("call_type")), + context=parent_context, + ) + if outer_ctx_data is not None: + self._store_open_call( + call_id, + _LLMCallSpan(span=None, start_time_ns=start_time_ns, outer_ctx_data=outer_ctx_data), + ) + return if not is_recordable_span(get_current_span(parent_context)): self._store_open_call(call_id, _LLMCallSpan(span=None, start_time_ns=start_time_ns)) return @@ -368,8 +385,9 @@ class OpenTelemetryV2(CustomLogger): return if self._emit_mcp_list_tools(kwargs, start_time, end_time): return - self._close_llm_call(kwargs, start_time, end_time) - self._record_metrics(kwargs, response_obj, start_time, end_time) + _, suppressed = self._close_llm_call_with_suppression(kwargs, start_time, end_time) + if not suppressed: + self._record_metrics(kwargs, response_obj, start_time, end_time) def _record_metrics(self, kwargs, response_obj, start_time, end_time) -> None: """Record the GenAI metrics for a successful LLM call.""" @@ -416,8 +434,9 @@ class OpenTelemetryV2(CustomLogger): return if self._emit_mcp_list_tools(kwargs, start_time, end_time): return - self._close_llm_call(kwargs, start_time, end_time) - self._record_failure_metrics(kwargs, start_time, end_time) + _, suppressed = self._close_llm_call_with_suppression(kwargs, start_time, end_time) + if not suppressed: + self._record_failure_metrics(kwargs, start_time, end_time) def _seed_identity_baggage(self, identity: RequestIdentity, model: str | None, context: Context) -> Context: """Seed authenticated request-identity Baggage onto ``context`` so the Baggage @@ -526,17 +545,25 @@ class OpenTelemetryV2(CustomLogger): start_time: datetime | float | None, end_time: datetime | float | None, ) -> Span | None: - """Finish the LLM-call span opened at ``pre_call`` (or create it deferred). + """Finish the LLM-call span opened at ``pre_call`` (or create it deferred).""" + span, _ = self._close_llm_call_with_suppression(kwargs, start_time, end_time) + return span - No carrier for this call id means ``pre_call`` never ran — the request was - rejected at the gate or blocked by a pre-call guardrail before any upstream - call — so there is nothing to record and no phantom span. - """ + def _close_llm_call_with_suppression( + self, + kwargs: Mapping[str, object], + start_time: datetime | float | None, + end_time: datetime | float | None, + ) -> tuple[Span | None, bool]: + """Finish the LLM-call span opened at ``pre_call`` (or create it deferred), + returning ``(span, suppressed)`` where ``suppressed`` is True when an outer + GenAI instrumentation owned the operation.""" call: Final = LLMCallEvent.from_dict(kwargs) call_id: Final = call.call_id # ``pop`` is the dedup: this method runs from both the success and failure # paths, and whichever fires first removes the carrier and closes the span. carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None + outer_ctx_data: Final = self._resolve_outer_genai_context(carrier, call, kwargs) # A missing carrier does not always mean nothing happened: a team/key-scoped # logger is a success/failure callback only, so ``pre_call`` never reaches it # and no carrier exists. The payload plus the request-level provider-handoff @@ -544,15 +571,37 @@ class OpenTelemetryV2(CustomLogger): # gate rejection carries ``is_no_upstream_call`` and gets no span. A cache # hit never hands off to a provider, so ``cache_hit`` substitutes for the # upstream stamp, since the payload proves the call completed. - if carrier is None and not call.reached_a_provider: - return None + if carrier is None and not call.reached_a_provider and outer_ctx_data is None: + return None, False try: - return self._finish_carrier(carrier, call, start_time, end_time) + span = self._finish_carrier( + carrier, + call, + start_time, + end_time, + kwargs=kwargs, + outer_ctx_data=outer_ctx_data, + ) + return span, outer_ctx_data is not None finally: # After the span has ended, so a release-triggered provider shutdown # force-flushes it out rather than racing its enqueue. self._release_carrier(carrier) + def _resolve_outer_genai_context( + self, + carrier: "_LLMCallSpan | None", + call: LLMCallEvent, + kwargs: Mapping[str, object], + ) -> object | None: + if carrier is not None and carrier.outer_ctx_data is not None: + return carrier.outer_ctx_data + return get_active_genai_context_data( + call.operation, + call_type=as_str(kwargs.get("call_type")), + context=_llm_call_parent_context(call), + ) + def _store_open_call(self, call_id: str, carrier: _LLMCallSpan) -> None: """Remember an in-flight LLM call, evicting the oldest if over budget. @@ -579,6 +628,9 @@ class OpenTelemetryV2(CustomLogger): call: LLMCallEvent, start_time: datetime | float | None, end_time: datetime | float | None, + *, + kwargs: Mapping[str, object] | None = None, + outer_ctx_data: object | None = None, ) -> Span | None: payload: Final = call.payload call_id: Final = call.call_id @@ -603,6 +655,22 @@ class OpenTelemetryV2(CustomLogger): metadata_keys=tuple(self.config.baggage_metadata_keys), operation=call.operation, ) + if outer_ctx_data is not None: + if self._emitter._seen(call_id, SpanRole.LLM_CALL): + return None + mapped: Final = self._emitter.map_attributes(data) + metric_attrs: Final = ( + self._metrics_recorder.bounded_attributes_for_context(kwargs) + if self._metrics_recorder is not None and kwargs is not None + else None + ) + publish_to_genai_context( + outer_ctx_data, # type: ignore[arg-type] + data, + mapped, + metric_attrs, + ) + return None end_time_ns: Final = to_ns(end_time) if carrier is not None and carrier.span is not None: # Born at the boundary: stamp attributes from the typed payload, set diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 1197ae055d0..af8cd70d5eb 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -366,6 +366,11 @@ class GenAIMetricRecorder: return {k: v for k, v in attrs.items() if k not in self._exclude} return attrs + def bounded_attributes_for_context(self, kwargs: Mapping[str, object]) -> dict[str, str]: + """Return bounded, filtered metric attributes suitable for publishing to an outer GenAI context data object.""" + filtered = self._filter_attributes(self._bounded_attributes(kwargs)) + return {k: v for k, v in filtered.items() if isinstance(v, str) and v} + # ------------------------------------------------------------------ # # Per-metric recording # ------------------------------------------------------------------ # diff --git a/pyproject.toml b/pyproject.toml index f2774e87b7b..6597c2c373a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,6 +185,7 @@ proxy-runtime = [ "opentelemetry-sdk==1.45.1", "opentelemetry-exporter-otlp==1.45.1", "opentelemetry-instrumentation-fastapi==0.66b1", + "opentelemetry-util-genai>=1.4b0", "ddtrace>=4.8.2,<5.0", "sentry-sdk>=2.21.0,<3.0", "mangum>=0.17.0,<1.0", @@ -231,6 +232,7 @@ dev = [ "opentelemetry-sdk==1.45.1", "opentelemetry-exporter-otlp==1.45.1", "opentelemetry-instrumentation-fastapi==0.66b1", + "opentelemetry-util-genai>=1.4b0", "langfuse>=4.7,<5.0", "fastapi-offline==1.7.6", "fakeredis==2.34.1", @@ -263,6 +265,7 @@ proxy-dev = [ "opentelemetry-sdk==1.45.1", "opentelemetry-exporter-otlp==1.45.1", "opentelemetry-instrumentation-fastapi==0.66b1", + "opentelemetry-util-genai>=1.4b0", "azure-identity==1.25.2", "a2a-sdk==1.1.0", "backports-zstd==1.8.0; python_version < '3.14'", diff --git a/tests/unit/integrations/otel/test_otel_genai_context_suppression.py b/tests/unit/integrations/otel/test_otel_genai_context_suppression.py new file mode 100644 index 00000000000..aa182cb0ece --- /dev/null +++ b/tests/unit/integrations/otel/test_otel_genai_context_suppression.py @@ -0,0 +1,407 @@ +"""Tests for OpenTelemetry GenAI client suppression and context attribute propagation. + +Covers LiteLLM as the inner layer per open-telemetry/semantic-conventions-genai#682 and +open-telemetry/opentelemetry-python-genai#859: +When an outer framework instrumentation (e.g. LangChain, CrewAI) has already +attached an ``InferenceData`` / ``EmbeddingData`` in ``opentelemetry.context`` +under ``CLIENT__CONTEXT_KEY``, LiteLLM suppresses its duplicate client span, +metrics, and events and publishes its resolved attributes into the active +context data object. +""" + +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone + +import pytest +from opentelemetry.context import attach, detach, set_value +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from litellm.integrations.otel.genai_context import ( + CLIENT_EMBEDDING_CONTEXT_KEY, + CLIENT_INFERENCE_CONTEXT_KEY, + EmbeddingData, + InferenceData, +) +from litellm.integrations.otel.logger import OpenTelemetryV2 +from litellm.integrations.otel.model.config import OpenTelemetryV2Config +from litellm.integrations.otel.model.semconv import Error, GenAI, LiteLLM, Server +from litellm.types.utils import StandardLoggingModelInformation, StandardLoggingPayload + + +def _make_payload( + *, + call_id: str = "call-123", + call_type: str = "completion", + model: str = "gpt-4o", + custom_llm_provider: str = "openai", + api_base: str = "https://api.openai.com:443/v1", + prompt_tokens: int = 12, + completion_tokens: int = 8, + response_cost: float = 0.0015, + status: str = "success", + error_str: str | None = None, + error_information: dict | None = None, +) -> StandardLoggingPayload: + now = datetime.now(timezone.utc) + return StandardLoggingPayload( + id="chatcmpl-xyz789", + trace_id="trace-1", + call_type=call_type, + stream=False, + response_cost=response_cost, + status=status, # type: ignore[typeddict-item] + custom_llm_provider=custom_llm_provider, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + startTime=now.timestamp(), + endTime=(now + timedelta(milliseconds=250)).timestamp(), + completionStartTime=now.timestamp(), + response_time=0.25, + model_map_information=StandardLoggingModelInformation( + model_map_key=model, + model_map_value=model, + ), + model=model, + model_id="deployment-1", + model_group=model, + api_base=api_base, + metadata={ + "user_api_key_hash": "hash-abc", + "user_api_key_team_id": "team-1", + "user_api_key_user_id": None, + "user_api_key_org_id": None, + "user_api_key_team_alias": "core-team", + "user_api_key_alias": "key-alias", + "user_api_key_user_email": None, + "user_api_key_end_user_id": None, + "spend_logs_metadata": None, + "requester_ip_address": None, + "requester_metadata": None, + "requester_custom_headers": None, + "prompt_management_metadata": None, + "applied_guardrails": None, + "mcp_tool_call_metadata": None, + "vector_store_request_metadata": None, + "usage_object": None, + "model_info": None, + "guardrail_information": None, + }, + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address=None, + messages=[{"role": "user", "content": "Hello world"}], + response={ + "id": "chatcmpl-xyz789", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hi there!"}}], + }, + error_str=error_str, + error_information=error_information, # type: ignore[typeddict-item] + model_parameters={ + "temperature": 0.5, + "max_tokens": 100, + "tools": [{"type": "function", "function": {"name": "get_weather", "description": "Get weather"}}], + }, + hidden_params={"model_id": "deployment-1", "litellm_model_name": model}, + litellm_call_id=call_id, + ) + + +def _make_v2_harness(*, capture_span_content: bool = False): + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + log_exporter = InMemoryLogExporter() + logger_provider = LoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + + cfg = OpenTelemetryV2Config( + exporter="in_memory", + enable_metrics=True, + enable_events=True, + capture_message_content="span_and_event" if capture_span_content else "no_content", + ) + logger = OpenTelemetryV2( + config=cfg, + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + return logger, span_exporter, metric_reader, log_exporter, tracer_provider, meter_provider, logger_provider + + +def _collected_metric_names(metric_reader: InMemoryMetricReader) -> list[str]: + data = metric_reader.get_metrics_data() + if data is None: + return [] + names: list[str] = [] + for rm in data.resource_metrics: + for sm in rm.scope_metrics: + for metric in sm.metrics: + names.append(metric.name) + return names + + +# =========================================================================== # +# 1. LiteLLM V2 as Inner Layer (Suppressed by Outer GenAI Context) +# =========================================================================== # + + +def test_v2_suppressed_when_outer_inference_context_active(): + """When CLIENT_INFERENCE_CONTEXT_KEY holds an InferenceData, OpenTelemetryV2 + suppresses its own client span and metrics and enriches the outer InferenceData.""" + logger, span_exporter, metric_reader, log_exporter, *_ = _make_v2_harness(capture_span_content=True) + + outer_data = InferenceData(request_model="langchain-alias") + token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data)) + try: + call_id = "call-suppressed-v2" + payload = _make_payload(call_id=call_id, model="gpt-4o", custom_llm_provider="openai") + kwargs = { + "litellm_call_id": call_id, + "call_type": "completion", + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello world"}], + "optional_params": { + "temperature": 0.5, + "max_tokens": 100, + "tools": [{"type": "function", "function": {"name": "get_weather", "description": "Get weather"}}], + }, + "litellm_params": {"custom_llm_provider": "openai", "metadata": {}}, + "standard_logging_object": payload, + "response_cost": 0.0015, + } + response_obj = { + "id": "chatcmpl-xyz789", + "model": "gpt-4o-2024-08-06", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hi there!"}}], + "usage": {"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20}, + } + now = datetime.now(timezone.utc) + + logger.log_pre_api_call("gpt-4o", kwargs["messages"], kwargs) + asyncio.run( + logger.async_log_success_event( + kwargs, + response_obj, + now, + now + timedelta(milliseconds=250), + ) + ) + finally: + detach(token) + + # 1. No span, metrics, or events emitted by LiteLLM + assert span_exporter.get_finished_spans() == () + assert _collected_metric_names(metric_reader) == [] + assert log_exporter.get_finished_logs() == () + + # 2. Outer InferenceData is enriched via attributes and metric_attributes dicts + assert outer_data.attributes.get(GenAI.PROVIDER_NAME) == "openai" + assert outer_data.attributes.get(GenAI.OPERATION_NAME) == "chat" + assert outer_data.attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o" + assert outer_data.attributes.get(GenAI.REQUEST_TEMPERATURE) == 0.5 + assert outer_data.attributes.get(GenAI.REQUEST_MAX_TOKENS) == 100 + assert outer_data.attributes.get(GenAI.RESPONSE_ID) == "chatcmpl-xyz789" + assert list(outer_data.attributes.get(GenAI.RESPONSE_FINISH_REASONS) or []) == ["stop"] + assert outer_data.attributes.get(GenAI.USAGE_INPUT_TOKENS) == 12 + assert outer_data.attributes.get(GenAI.USAGE_OUTPUT_TOKENS) == 8 + assert outer_data.attributes.get(Server.ADDRESS) == "api.openai.com" + assert outer_data.attributes.get(Server.PORT) == 443 + + # Custom LiteLLM attributes published to outer_data.attributes + assert outer_data.attributes.get(LiteLLM.CALL_ID) == call_id + assert outer_data.attributes.get(LiteLLM.CACHE_HIT) is False + + # Metric attributes published to outer_data.metric_attributes + assert outer_data.metric_attributes.get(GenAI.PROVIDER_NAME) == "openai" + assert outer_data.metric_attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o" + + # Content capture attributes published into outer_data.attributes + assert outer_data.attributes.get(GenAI.INPUT_MESSAGES) is not None + assert outer_data.attributes.get(GenAI.OUTPUT_MESSAGES) is not None + + +def test_v2_suppressed_failure_enriches_outer_inference_context(): + """On a failed LLM call when suppressed, OpenTelemetryV2 suppresses its error span, + failure duration metric, and exception event, and publishes error details to outer_data.""" + logger, span_exporter, metric_reader, log_exporter, *_ = _make_v2_harness() + + outer_data = InferenceData() + token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data)) + try: + call_id = "call-fail-suppressed-v2" + payload = _make_payload( + call_id=call_id, + model="gpt-4o", + custom_llm_provider="openai", + status="failure", + error_str="Rate limit exceeded", + error_information={ + "error_code": "429", + "error_class": "RateLimitError", + "error_message": "Rate limit exceeded", + "llm_provider": "openai", + "traceback": "Traceback ...", + }, + ) + kwargs = { + "litellm_call_id": call_id, + "call_type": "completion", + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai", "metadata": {}}, + "standard_logging_object": payload, + "exception": RuntimeError("Rate limit exceeded"), + } + now = datetime.now(timezone.utc) + + logger.log_pre_api_call("gpt-4o", kwargs["messages"], kwargs) + asyncio.run( + logger.async_log_failure_event( + kwargs, + None, + now, + now + timedelta(milliseconds=100), + ) + ) + finally: + detach(token) + + assert span_exporter.get_finished_spans() == () + assert _collected_metric_names(metric_reader) == [] + assert log_exporter.get_finished_logs() == () + + assert outer_data.attributes.get(Error.TYPE) == "RateLimitError" + assert outer_data.attributes.get(Error.MESSAGE) == "Rate limit exceeded" + + +def test_v2_suppressed_with_real_opentelemetry_util_genai_handler(): + """End-to-end test with opentelemetry.util.genai.handler.TelemetryHandler as the outer layer.""" + util_genai = pytest.importorskip("opentelemetry.util.genai.handler") + + ( + logger, + span_exporter, + _metric_reader, + _log_exporter, + tracer_provider, + meter_provider, + logger_provider, + ) = _make_v2_harness(capture_span_content=True) + + handler = util_genai.TelemetryHandler( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + + call_id = "call-e2e-outer-handler" + payload = _make_payload( + call_id=call_id, + model="claude-3-5-sonnet", + custom_llm_provider="anthropic", + prompt_tokens=15, + completion_tokens=9, + ) + kwargs = { + "litellm_call_id": call_id, + "call_type": "completion", + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "Hello Claude"}], + "optional_params": {"temperature": 0.2}, + "litellm_params": {"custom_llm_provider": "anthropic", "metadata": {}}, + "standard_logging_object": payload, + "response_cost": 0.003, + } + response_obj = { + "id": "msg-01", + "model": "claude-3-5-sonnet-20241022", + "choices": [{"finish_reason": "end_turn", "message": {"role": "assistant", "content": "Hello!"}}], + "usage": {"prompt_tokens": 15, "completion_tokens": 9, "total_tokens": 24}, + } + now = datetime.now(timezone.utc) + + with handler.inference(None, request_model="outer-model"): + logger.log_pre_api_call("claude-3-5-sonnet", kwargs["messages"], kwargs) + asyncio.run( + logger.async_log_success_event( + kwargs, + response_obj, + now, + now + timedelta(milliseconds=200), + ) + ) + + spans = span_exporter.get_finished_spans() + # Exactly ONE client span is emitted (by the outer TelemetryHandler, enriched by LiteLLM) + assert len(spans) == 1 + outer_span = spans[0] + attrs = dict(outer_span.attributes or {}) + assert attrs.get(GenAI.PROVIDER_NAME) == "anthropic" + assert attrs.get(GenAI.USAGE_INPUT_TOKENS) == 15 + assert attrs.get(GenAI.USAGE_OUTPUT_TOKENS) == 9 + assert attrs.get(LiteLLM.CALL_ID) == call_id + + +def test_v2_embedding_context_suppression(): + """Embedding calls use CLIENT_EMBEDDING_CONTEXT_KEY and EmbeddingData.""" + logger, span_exporter, metric_reader, *_ = _make_v2_harness() + + outer_emb = EmbeddingData(request_model="text-embedding-3-small") + token = attach(set_value(CLIENT_EMBEDDING_CONTEXT_KEY, outer_emb)) + try: + call_id = "call-emb-suppressed" + payload = _make_payload( + call_id=call_id, + call_type="embedding", + model="text-embedding-3-small", + custom_llm_provider="openai", + prompt_tokens=10, + completion_tokens=0, + ) + kwargs = { + "litellm_call_id": call_id, + "call_type": "embedding", + "model": "text-embedding-3-small", + "messages": ["embed me"], + "optional_params": {"encoding_format": "float"}, + "litellm_params": {"custom_llm_provider": "openai", "metadata": {}}, + "standard_logging_object": payload, + } + now = datetime.now(timezone.utc) + logger.log_pre_api_call("text-embedding-3-small", kwargs["messages"], kwargs) + asyncio.run( + logger.async_log_success_event( + kwargs, + None, + now, + now + timedelta(milliseconds=100), + ) + ) + finally: + detach(token) + + assert span_exporter.get_finished_spans() == () + assert _collected_metric_names(metric_reader) == [] + assert outer_emb.attributes.get(GenAI.PROVIDER_NAME) == "openai" + assert outer_emb.attributes.get(GenAI.REQUEST_MODEL) == "text-embedding-3-small" + assert outer_emb.attributes.get(GenAI.USAGE_INPUT_TOKENS) == 10 + +